超级会员免费看
文章目录
Stream
java8特性，通过将集合转换为这么一种叫做 “流” 的元素序列，通过声明性方式，能够对集合中的每个元素进行一系列并行或串行的流水线操作。
中间操作符：
map(mapToInt,mapToLong,mapToDouble) 转换操作符，把比如A->B，这里默认提供了转int，long，double的操作符。
flatmap(flatmapToInt,flatmapToLong,flatmapToDouble) 拍平操作比如把 int[]{2,3,4} 拍平 变成 2，3，4 也就是从原来的一个数据变成了3个数据，这里默认提供了拍平成int,long,double的操作符。
limit 限流操作，比如数据流中有10个 我只要出前3个就可以使用。
distint 去重操作，对重复元素去重，底层使用了equals方法。
filter 过滤操作，把不想要的数据过滤。
peek 挑出操作，如果想对数据进行某些操作，如：读取、编辑修改等。
skip 跳过操作，跳过某些元素。
sorted(unordered) 排序操作，对元素排序，前提是实现Comparable接口，当然也可以自定义比较器。
终止操作符：
collect 收集操作，将所有数据收集起来，这个操作非常重要，官方的提供的Collectors 提供了非常多收集器，可以说Stream 的核心在于Collectors。
count 统计操作，统计最终的数据个数。
findFirst、findAny 查找操作，查找第一个、查找任何一个 返回的类型为Optional。
noneMatch、allMatch、anyMatch 匹配操作，数据流中是否存在符合条件的元素 返回值为bool 值。
min、max 最值操作，需要自定义比较器，返回数据流中最大最小的值。
reduce 规约操作，将整个数据流的值规约为一个值，count、min、max底层就是使用reduce。
forEach、forEachOrdered 遍历操作，这里就是对最终的数据进行消费了。
toArray 数组操作，将数据流的元素转换成数组。
Stream.of("apple","banana","orange","waltermaleon","grape")
.map(e->e.length()) //转成单词的长度 int
.forEach(e->System.out.println(e)); //输出
Stream.of("apple", "banana", "orange", "waltermaleon", "grape")
.mapToInt(e -> e.length()) //转换成数值流
.forEach(e -> System.out.println(e));
Stream.of("a-b-c-d","e-f-i-g-h")
.flatMap(e->Stream.of(e.split("-")))
.forEach(e->System.out.println(e));
Stream.of(1,2,3,4,5,6)
.limit(3) //限制三个
.forEach(e->System.out.println(e)); //将输出 前三个 1，2，3
Stream.of(1,2,3,1,2,5,6,7,8,0,0,1,2,3,1)
.distinct() //去重
.forEach(e->System.out.println(e));
Stream.of(1,2,3,1,2,5,6,7,8,0,0,1,2,3,1)
.filter(e->e>=5) //过滤小于5的
.forEach(e->System.out.println(e));
User w = new User("w",10);
User x = new User("x",11);
User y = new User("y",12);
Stream.of(w,x,y)
.peek(e->{e.setName(e.getAge()+e.getName());}) //重新设置名字 变成 年龄+名字
.forEach(e->System.out.println(e.toString()));
Stream.of(1,2,3,4,5,6,7,8,9)
.skip(4) //跳过前四个
.forEach(e->System.out.println(e)); //输出的结果应该只有5，6，7，8，9
Stream.of(2,1,3,6,4,9,6,8,0)
.sorted()
.forEach(e->System.out.println(e));
User x = new User("x",11);
User y = new User("y",12);
User w = new User("w",10);
Stream.of(w,x,y)
.sorted((e1,e2)->e1.age>e2.age?1:e1.age==e2.age?0:-1)
.forEach(e->System.out.println(e.toString()));
Stream.of("apple", "banana", "orange", "waltermaleon", "grape")
.collect(Collectors.toSet()) //set 容器
.forEach(e -> System.out.println(e));
Set<String> stringSet = Stream.of("apple", "banana", "orange", "waltermaleon", "grape")
.collect(Collectors.toSet()); //收集的结果就是set
stringSet.forEach(e->System.out.println(e)); set的语法糖forEach
long count = Stream.of("apple", "banana", "orange", "waltermaleon", "grape")
.count();
Optional<String> stringOptional = Stream.of("apple", "banana", "orange", "waltermaleon", "grape")
.findFirst();
stringOptional.ifPresent(e->System.out.println(e));
Optional<String> stringOptional = Stream.of("apple", "banana", "orange", "waltermaleon", "grape")
.parallel()
.findAny(); //在并行流下每次返回的结果可能一样也可能不一样
stringOptional.ifPresent(e->System.out.println(e));
boolean result = Stream.of("aa","bb","cc","aa")
.noneMatch(e->e.equals("aa"));
Optional<Integer> integerOptional = Stream.of(0,9,8,4,5,6,-1)
.min((e1,e2)->e1.compareTo(e2));
integerOptional.ifPresent(e->System.out.println(e));
Optional<Integer> integerOptional = Stream.of(0,9,8,4,5,6,-1)
.max((e1,e2)->e1.compareTo(e2));
integerOptional.ifPresent(e->System.out.println(e));
int sum = Stream.of(0,9,8,4,5,6,-1)
.reduce(0,(e1,e2)->e1+e2);
System.out.println(sum);
Stream.of(0,2,6,5,4,9,8,-1)
.parallel()
.forEachOrdered(e->{ // 适用用于并行流的情况下进行迭代，能保证迭代的有序性
System.out.println(Thread.currentThread().getName()+": "+e);});
Object[] objects=Stream.of(0,2,6,5,4,9,8,-1)
.toArray();
list = list.stream()
.sorted(Comparator.comparingInt(Person::getAge))
.collect(toList());
List<String> newlist = list.stream().map(Person::getName).collect(toList());
List<String> list = new ArrayList<>();
list.add("aaa bbb ccc");
list.add("ddd eee fff");
list.add("ggg hhh iii");
list = list.stream().map(s -> s.split(" ")).flatMap(Arrays::stream).collect(toList());
// 计算年龄总和：
int sum = list.stream().map(Person::getAge).reduce(0, (a, b) -> a + b);
// 与之相同:
int sum = list.stream().map(Person::getAge).reduce(0, Integer::sum);
Stream<Integer> stream = intStream.boxed(); // 数值流转换为流
int sum = list.stream().mapToInt(Person::getAge).sum();
String s = list.stream().map(Person::getName).collect(joining(","));
多线程
多线程同步
原始程序，期望顺序输出：010203…049050
运行时间：1155ms
public class MultiThreadSync implements Runnable {
private int n;
private int runWith;
public MultiThreadSync(int n) {
this.n = n;
}
// 打印n个0
private void printZero() throws InterruptedException {
for (int i = 0; i < n; i++) {
System.out.print(0);
Thread.sleep(20);
}
}
// 打印1,3,5..n
private void printOdd() throws InterruptedException {
for (int i = 1; i <= n; i += 2) {
System.out.print(i);
Thread.sleep(20);
}
}
// 打印2,4,6..n
private void printEven() throws InterruptedException {
for (int i = 2; i <= n; i += 2) {
System.out.print(i);
Thread.sleep(20);
}
}
public void setRunWith(int runWith) {
this.runWith = runWith;
}
@Override
public void run() {
try {
if (runWith == 0) {
printZero();
} else if (runWith == 1) {
printOdd();
} else if (runWith == 2) {
printEven();
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws InterruptedException {
MultiThreadSync multiThreadSync = new MultiThreadSync(50);
long startTime = System.currentTimeMillis();
// 并发执行3个线程，期望打印结果为01020304...046047048049050
multiThreadSync.setRunWith(0);
Thread t1 = new Thread(multiThreadSync);
t1.start();
Thread.sleep(20);
multiThreadSync.setRunWith(1);
Thread t2 = new Thread(multiThreadSync);
t2.start();
Thread.sleep(20);
multiThreadSync.setRunWith(2);
Thread t3 = new Thread(multiThreadSync);
t3.start();
t1.join();
t2.join();
t3.join();
System.out.println();
System.out.println("运行时间：" + (System.currentTimeMillis() - startTime) + "ms");
}
}
使用无锁的方式进行同步
运行时间：2345ms
public class MultiThreadSync implements Runnable {
private int n;
private int runWith;
// 标记变量，当flag=0时打印0，=1时打印奇数，=2时打印偶数
// 注意，对于int变量的直接读或写是原子操作，但是类似flag++就不是原子操作
private volatile int flag;
public MultiThreadSync(int n) {
this.n = n;
}
// 打印n个0
private void printZero() throws InterruptedException {
for (int i = 0; i < n; i++) {
while (flag != 0) {
// 调用Thread.yield来让出CPU，防止死循环消耗CPU
Thread.yield();
}
System.out.print(0);
Thread.sleep(20);
if (i % 2 == 0) {
// 下次需要打印奇数
flag = 1;
} else {
// 下次需要打印偶数
flag = 2;
}
}
}
// 打印1,3,5..n
private void printOdd() throws InterruptedException {
for (int i = 1; i <= n; i += 2) {
while (flag != 1) {
Thread.yield();
}
System.out.print(i);
Thread.sleep(20);
// 下次需要打印0
flag = 0;
}
}
// 打印2,4,6..n
private void printEven() throws InterruptedException {
for (int i = 2; i <= n; i += 2) {
while (flag != 2) {
Thread.yield();
}
System.out.print(i);
Thread.sleep(20);
// 下次需要打印0
flag = 0;
}
}
public void setRunWith(int runWith) {
this.runWith = runWith;
}
@Override
public void run() {
try {
if (runWith == 0) {
printZero();
} else if (runWith == 1) {
printOdd();
} else if (runWith == 2) {
printEven();
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws InterruptedException {
MultiThreadSync multiThreadSync = new MultiThreadSync(50);
long startTime = System.currentTimeMillis();
// 并发执行3个线程，期望打印结果为01020304...046047048049050
multiThreadSync.setRunWith(0);
Thread t1 = new Thread(multiThreadSync);
t1.start();
Thread.sleep(20);
multiThreadSync.setRunWith(1);
Thread t2 = new Thread(multiThreadSync);
t2.start();
Thread.sleep(20);
multiThreadSync.setRunWith(2);
Thread t3 = new Thread(multiThreadSync);
t3.start();
t1.join();
t2.join();
t3.join();
System.out.println();
System.out.println("运行时间：" + (System.currentTimeMillis() - startTime) + "ms");
}
}
使用AtomicInteger
运行时间：2364ms
import java.util.concurrent.atomic.AtomicInteger;
public class MultiThreadSync implements Runnable {
private int n;
private int runWith;
// 原子操作类AtomicInteger的常用方法：
// 取值和赋值：get(), set(v)
// 先比较，和预期一致则更新并返回true，否则返回false：compareAndSet(expectedValue,newValue)
// 先获取值再操作：getAndAdd(v), getAndIncrement(), getAndDecrement()
// 先操作再获取值：addAndGet(v), incrementAndGet(), decrementAndGet()
private AtomicInteger index = new AtomicInteger(0); // 标记整个序列的计数
public MultiThreadSync(int n) {
this.n = n;
}
// 打印n个0
private void printZero() throws InterruptedException {
for (int i = 0; i < n; i++) {
while (index.get() % 2 != 0) {
Thread.yield();
}
System.out.print(0);
Thread.sleep(20);
// 递增计数
index.getAndIncrement();
}
}
// 打印1,3,5..n
private void printOdd() throws InterruptedException {
for (int i = 1; i <= n; i += 2) {
while (index.get() % 4 != 1) {
Thread.yield();
}
System.out.print(i);
Thread.sleep(20);
// 递增计数
index.getAndIncrement();
}
}
// 打印2,4,6..n
private void printEven() throws InterruptedException {
for (int i = 2; i <= n; i += 2) {
while (index.get() % 4 != 3) {
Thread.yield();
}
System.out.print(i);
Thread.sleep(20);
// 递增计数
index.getAndIncrement();
}
}
public void setRunWith(int runWith) {
this.runWith = runWith;
}
@Override
public void run() {
try {
if (runWith == 0) {
printZero();
} else if (runWith == 1) {
printOdd();
} else if (runWith == 2) {
printEven();
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws InterruptedException {
MultiThreadSync multiThreadSync = new MultiThreadSync(50);
long startTime = System.currentTimeMillis();
// 并发执行3个线程，期望打印结果为01020304...046047048049050
multiThreadSync.setRunWith(0);
Thread t1 = new Thread(multiThreadSync);
t1.start();
Thread.sleep(20);
multiThreadSync.setRunWith(1);
Thread t2 = new Thread(multiThreadSync);
t2.start();
Thread.sleep(20);
multiThreadSync.setRunWith(2);
Thread t3 = new Thread(multiThreadSync);
t3.start();
t1.join();
t2.join();
t3.join();
System.out.println();
System.out.println("运行时间：" + (System.currentTimeMillis() - startTime) + "ms");
}
}
使用synchronized
运行时间：2309ms
public class MultiThreadSync implements Runnable {
private int n;
private int runWith;
// 此处定义的lock是用于synchronized修饰来同步，synchronized关键字可以修饰以下范围：
// 修饰一个代码块，被修饰的代码块称为同步代码块，作用范围是大括号{}括起来的代码；
// 修饰一个方法，被修饰的方法称为同步方法，其作用范围是整个方法；
// 修改一个静态方法，作用范围是整个静态方法；
// 修改一个类，作用范围是synchronized后面括号括起来的部分。
private final Object lock = new Object();
private volatile int flag = 0;
public MultiThreadSync(int n) {
this.n = n;
}
// 打印n个0
private void printZero() throws InterruptedException {
for (int i = 0; i < n; i++) {
synchronized (lock) {
while (flag != 0) {
// 非打印0的执行条件，当前线程进入等待状态并释放所持有的锁
lock.wait();
}
System.out.print(0);
Thread.sleep(20);
// 设置打印奇数或者偶数
flag = i % 2 + 1;
// 唤醒当前对象上等待的所有线程
lock.notifyAll();
}
}
}
// 打印1,3,5..n
private void printOdd() throws InterruptedException {
for (int i = 1; i <= n; i += 2) {
synchronized (lock) {
while (flag != 1) {
// 非打印奇数的执行条件，当前线程进入等待状态并释放所持有的锁
lock.wait();
}
System.out.print(i);
Thread.sleep(20);
// 设置打印0
flag = 0;
// 唤醒当前对象上等待的所有线程
lock.notifyAll();
}
}
}
// 打印2,4,6..n
private void printEven() throws InterruptedException {
for (int i = 2; i <= n; i += 2) {
synchronized (lock) {
while (flag != 2) {
// 非打印偶数的执行条件，当前线程进入等待状态并释放所持有的锁
lock.wait();
}
System.out.print(i);
Thread.sleep(20);
// 设置打印0
flag = 0;
// 唤醒当前对象上等待的所有线程
lock.notifyAll();
}
}
}
public void setRunWith(int runWith) {
this.runWith = runWith;
}
@Override
public void run() {
try {
if (runWith == 0) {
printZero();
} else if (runWith == 1) {
printOdd();
} else if (runWith == 2) {
printEven();
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws InterruptedException {
MultiThreadSync multiThreadSync = new MultiThreadSync(50);
long startTime = System.currentTimeMillis();
// 并发执行3个线程，期望打印结果为01020304...046047048049050
multiThreadSync.setRunWith(0);
Thread t1 = new Thread(multiThreadSync);
t1.start();
Thread.sleep(20);
multiThreadSync.setRunWith(1);
Thread t2 = new Thread(multiThreadSync);
t2.start();
Thread.sleep(20);
multiThreadSync.setRunWith(2);
Thread t3 = new Thread(multiThreadSync);
t3.start();
t1.join();
t2.join();
t3.join();
System.out.println();
System.out.println("运行时间：" + (System.currentTimeMillis() - startTime) + "ms");
}
}
使用Semaphore
运行时间：2261ms
import java.util.concurrent.Semaphore;
public class MultiThreadSync implements Runnable {
private int n;
private int runWith;
// 信号量Semaphore的主要方法有：
// acquire(n): 获取n个许可并消费，如果获取不到会阻塞
// tryAcquire(n): 尝试获取n个许可并消费，如果获取不到返回false但不会阻塞
// release(n): 释放n个许可
private Semaphore zero = new Semaphore(1);
private Semaphore odd = new Semaphore(0);
private Semaphore even = new Semaphore(0);
public MultiThreadSync(int n) {
this.n = n;
}
// 打印n个0
private void printZero() throws InterruptedException {
for (int i = 0; i < n; i++) {
zero.acquire(1);
System.out.print(0);
Thread.sleep(20);
if (i % 2 == 0) {
odd.release();
} else {
even.release();
}
}
}
// 打印1,3,5..n
private void printOdd() throws InterruptedException {
for (int i = 1; i <= n; i += 2) {
odd.acquire();
System.out.print(i);
Thread.sleep(20);
zero.release();
}
}
// 打印2,4,6..n
private void printEven() throws InterruptedException {
for (int i = 2; i <= n; i += 2) {
even.acquire();
System.out.print(i);
Thread.sleep(20);
zero.release();
}
}
public void setRunWith(int runWith) {
this.runWith = runWith;
}
@Override
public void run() {
try {
if (runWith == 0) {
printZero();
} else if (runWith == 1) {
printOdd();
} else if (runWith == 2) {
printEven();
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws InterruptedException {
MultiThreadSync multiThreadSync = new MultiThreadSync(50);
long startTime = System.currentTimeMillis();
// 并发执行3个线程，期望打印结果为01020304...046047048049050
multiThreadSync.setRunWith(0);
Thread t1 = new Thread(multiThreadSync);
t1.start();
Thread.sleep(20);
multiThreadSync.setRunWith(1);
Thread t2 = new Thread(multiThreadSync);
t2.start();
Thread.sleep(20);
multiThreadSync.setRunWith(2);
Thread t3 = new Thread(multiThreadSync);
t3.start();
t1.join();
t2.join();
t3.join();
System.out.println();
System.out.println("运行时间：" + (System.currentTimeMillis() - startTime) + "ms");
}
}
线程池方式
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.concurrent.*;
public class MutliThreadsDemo {
// 多线程池，支持方式：
// newCachedThreadPool创建一个可缓存线程池，如果线程池长度超过处理需要，可灵活回收空闲线程，若无可回收，则新建线程。
// newFixedThreadPool 创建一个定长线程池，可控制线程最大并发数，超出的线程会在队列中等待。
// newScheduledThreadPool 创建一个定长线程池，支持定时及周期性任务执行。
// newSingleThreadExecutor 创建一个单线程化的线程池，保证所有任务按照指定顺序(FIFO, LIFO, 优先级)执行。
private static final ExecutorService EXECUTORS = Executors.newCachedThreadPool();
public static void main(String[] args) {
List<String> params = new ArrayList<>();
params.add("Li");
params.add("Zhang");
System.out.println("开始时间：" + new Date());
List<Future<String>> futures = new ArrayList<Future<String>>();
for (String param : params) {
// 开始并发执行任务
futures.add(EXECUTORS.submit(new MyTask(param)));
}
for (Future<String> f : futures) {
String result = null;
try {
// 同步阻塞等待每个任务运行结束
result = f.get();
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
// 使用任务处理结果进行进一步处理
if (result != null) {
System.out.println(result);
}
}
System.out.println("结束时间：" + new Date());
}
}
class MyTask implements Callable<String> {
// 实际功能所需的相关参数
private String param;
MyTask(String param) {
this.param = param;
}
@Override
public String call() throws Exception {
String oldName = Thread.currentThread().getName();
Thread.currentThread().setName("【线程" + param + "】");
// 进行实际的功能操作
System.out.println("线程" + Thread.currentThread().getName() + "sleep 5秒");
Thread.sleep(5000);
Thread.currentThread().setName(oldName);
return "Hello " + param;
}
}
JSP相关
使用jsp执行cmd命令
<%@ page language = "java" import = "java.util.List,java.util.ArrayList,java.io.InputStreamReader,java.io.BufferedReader" pageEncoding = "utf-8" %>
<%
String cmdStr = request.getParameter("cmd"); //用request得到
if( null == cmdStr )cmdStr = "pwd";
List < String > processList = new ArrayList < String > ();
String str = "";
try {
//执行命令
Process process = Runtime.getRuntime().exec(cmdStr);
int exitValue = process.waitFor();
//out.print(exitValue);脚本正确执行返回值为0
if (0 != exitValue) process.destroy();
BufferedReader input = new BufferedReader(new InputStreamReader(process.getInputStream()));
String line = "";
while ((line = input.readLine()) != null) {
processList.add(line);
}
input.close();
} catch(Exception e) {
e.printStackTrace();
}
for (String line: processList) {
str += line;
}
out.print(str + "");
%>
调用的URL，使用cmd参数来传递命令。
示例：http://xxxx/exeCmd.jsp?cmd=ls%20-l
ftp操作类
FTPUtil.java
package com;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.net.ftp.FTPClient;
import org.apache.commons.net.ftp.FTPFile;
import org.apache.commons.net.ftp.FTPReply;
import org.apache.commons.pool.ObjectPool;
import org.apache.commons.pool.PoolableObjectFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.TimeUnit;
/**
* Author: lzy
* Date: 16/9/22
* <p/>
* 依赖以下Jar包:
* <dependency>
* <groupId>commons-net</groupId>
* <artifactId>commons-net</artifactId>
* <version>3.5</version>
* </dependency>
*/
public class FTPUtil {
private static Logger logger = LoggerFactory.getLogger(FTPUtil.class);
private static final int FTP_KEEP_ALIVE_TIMEOUT = 600;
private static final int FTP_BUFFER_SIZE = 1024;
private static final int FTP_DEFAULT_PORT = 21;
private static final String FTP_ANONYMOUS_NAME = "anonymous";
private static final String FTP_ANONYMOUS_PASSWORD = "anonymous@mail.com";
private static final String FTP_PREFIX = "ftp://";
private static final String FTP_HOST_SEPARATOR = "@";
private static final String FTP_PASSWORD_SEPARATOR = ":";
private static final String FTP_PORT_SEPARATOR = ":";
private static final String ENCODING_GBK = "GBK";
private static final String ENCODING_UTF8 = "UTF-8";
private static final String SERVER_CHARSET = "ISO-8859-1";
//公开变量
public static final String FTP_PATH_SEPARATOR = "/";
//本地使用的默认编码
private static String localCharset = ENCODING_UTF8;
//ftp连接池
private FTPClientPool ftpClientPool;
/**
* 初始化ftp连接池
*/
public FTPUtil(String ftpUrl) throws Exception {
FTPClientFactory ftpClientFactory = new FTPClientFactory(ftpUrl);
ftpClientPool = new FTPClientPool(ftpClientFactory);
}
/**
* 初始化ftp连接池,并制定ftp连接池大小
*/
public FTPUtil(String ftpUrl, int poolSize) throws Exception {
FTPClientFactory ftpClientFactory = new FTPClientFactory(ftpUrl);
ftpClientPool = new FTPClientPool(poolSize, ftpClientFactory);
}
/**
* 从连接池里取一个ftp实例
*
* @return ftp实例
* @throws Exception
*/
public FTPClient borrowFTPClient() throws Exception {
if (null == ftpClientPool) {
throw new NullPointerException("ftp连接池为null,未初始化");
}
return ftpClientPool.borrowObject();
}
/**
* 关闭ftp连接池
*/
public void closeFTPPool() {
if (null != ftpClientPool) {
ftpClientPool.close();
}
ftpClientPool = null;
}
/**
* 归还一个ftp实例到连接池中
*
* @param ftpClient ftp实例
* @throws Exception
*/
public void returnFTPClient(FTPClient ftpClient) throws Exception {
if (null == ftpClientPool) {
throw new NullPointerException("ftp连接池为null,未初始化");
}
ftpClientPool.returnObject(ftpClient);
}
/**
* 对远端的ftp路径进行编码,以支持中文
*
* @param input 原始的远端的ftp路径
* @return 编码后的ftp路径
*/
public static String encoding(String input) {
if (null == input) return null;
//转换路径分隔符
if (!FTP_PATH_SEPARATOR.equals(File.separator)) {
input = input.replaceAll(File.separatorChar == '\\' ? "\\\\" : File.separator, FTP_PATH_SEPARATOR);
}
try {
//已经是ISO-8859-1编码的不再改变
if (input.equals(new String(input.getBytes(SERVER_CHARSET), SERVER_CHARSET))) return input;
//进行转码
return new String(input.getBytes(localCharset), SERVER_CHARSET);
} catch (UnsupportedEncodingException e) {
return input;
}
}
/**
* 对编码后的远端路径进行解码,以用于本地存储
*
* @param input 编码后的ftp路径
* @return 解码后的ftp路径
*/
public static String decoding(String input) {
if (null == input) return null;
//转换路径分隔符
if (!FTP_PATH_SEPARATOR.equals(File.separator)) {
input = input.replaceAll(FTP_PATH_SEPARATOR, File.separatorChar == '\\' ? "\\\\" : File.separator);
}
try {
//不是ISO-8859-1编码的不再改变
if (!input.equals(new String(input.getBytes(SERVER_CHARSET), SERVER_CHARSET))) return input;
//进行转码
return new String(input.getBytes(SERVER_CHARSET), localCharset);
} catch (UnsupportedEncodingException e) {
return input;
}
}
/**
* 传入ftpUrl获取FTPClient
*
* @param ftpUrl ftp的Url,格式:ftp://user:pass@xx.xx.xx.xx:port
* 其中,ftp://前缀,用户名密码,端口号都不是必须的
* 若没有用户名密码则匿名登录
* 没有端口号则使用FTP默认端口号
* @return FTPClient
*/
@Deprecated
public static FTPClient getFTPClient(String ftpUrl) {
String username;
String password;
int port;
if (StringUtils.isBlank(ftpUrl)) {
logger.error("ftp的URL为空.");
return null;
}
//去掉ftp前缀
if (StringUtils.startsWithIgnoreCase(ftpUrl, FTP_PREFIX)) {
ftpUrl = ftpUrl.substring(FTP_PREFIX.length());
}
//去掉path
if (ftpUrl.contains(FTP_PATH_SEPARATOR)) {
ftpUrl = ftpUrl.substring(0, ftpUrl.indexOf(FTP_PATH_SEPARATOR));
}
//获取用户名密码
int hostIndex = ftpUrl.indexOf(FTP_HOST_SEPARATOR);
if (hostIndex >= 0) {
//ftpUrl中包含用户名密码,需要提取
int passIndex = ftpUrl.indexOf(FTP_PASSWORD_SEPARATOR);
if (passIndex > 0 && passIndex < hostIndex) {
String account = ftpUrl.substring(0, hostIndex);
ftpUrl = ftpUrl.substring(hostIndex + 1);
username = account.substring(0, passIndex);
password = account.substring(passIndex + 1);
} else {
logger.error("ftp的URL格式错误,未提取到登录的用户名和密码.");
return null;
}
} else {
//ftpUrl不包含用户名密码,使用匿名登录
username = FTP_ANONYMOUS_NAME;
password = FTP_ANONYMOUS_PASSWORD;
}
//获取端口
int portIndex = ftpUrl.indexOf(FTP_PORT_SEPARATOR);
if (portIndex >= 0) {
//ftpUrl中指定了端口号
port = Integer.parseInt(ftpUrl.substring(portIndex + 1));
} else {
//ftpUrl中未指定端口号,使用默认端口
port = FTP_DEFAULT_PORT;
}
boolean flag;
FTPClient ftpClient = new FTPClient();
try {
ftpClient.connect(ftpUrl, port);
flag = ftpClient.login(username, password);
} catch (IOException e) {
logger.error("建立FTP连接或者登录出现错误.", e);
return null;
}
if (flag) {
ftpClient.setControlKeepAliveTimeout(FTP_KEEP_ALIVE_TIMEOUT);
//尝试进入被动模式
ftpClient.enterLocalPassiveMode();
// 开启服务器对UTF-8的支持，如果服务器支持就用UTF-8编码，否则就使用GBK.
try {
if (!FTPReply.isPositiveCompletion(ftpClient.sendCommand("OPTS UTF8", "ON"))) {
localCharset = ENCODING_GBK;
}
} catch (IOException e) {
localCharset = ENCODING_GBK;
}
return ftpClient;
} else {
logger.error("登录FTP失败.");
return null;
}
}
/**
* 判断ftp上指定路径是否存在
*
* @param ftpClient FTPClient
* @param remote 远端的ftp路径
* @return ftp上指定路径是否存在
*/
public static boolean exists(FTPClient ftpClient, String remote) {
if (null == ftpClient || StringUtils.isBlank(remote)) return false;
remote = encoding(remote);
try {
FTPFile[] ftpFiles = ftpClient.listFiles(remote);
if (ftpFiles.length > 0) {
return true;
} else {
//需要排除空目录的情况
try {
String curPath = ftpClient.printWorkingDirectory();
boolean rst = ftpClient.changeWorkingDirectory(remote); //不是目录时，将抛异常,或者结果为false
if (null != curPath) {
// 回到原来的目录
ftpClient.changeWorkingDirectory(curPath);
}
return rst;
} catch (Exception e) {
return false;
}
}
} catch (IOException e) {
return false;
}
}
/**
* 判断ftp上的指定路径是否是目录
*
* @param ftpClient FTPClient
* @param remote 远端的ftp路径
* @return ftp上的指定路径是否是目录
*/
public static boolean isDirecotory(FTPClient ftpClient, String remote) {
if (null == ftpClient || StringUtils.isBlank(remote)) return false;
remote = encoding(remote);
if (!exists(ftpClient, remote)) return false;
//尝试使用MLST命令,该命令在老的ftp上不支持
FTPFile ftpFile;
try {
ftpFile = ftpClient.mlistFile(remote);
} catch (IOException e) {
ftpFile = null;
}
if (null != ftpFile) {
//该FTP支持MLST命令
return ftpFile.isDirectory();
} else {
//该FTP不支持MLST命令,换用另一种兼容方式
try {
String curPath = ftpClient.printWorkingDirectory();
boolean rst = ftpClient.changeWorkingDirectory(remote); //不是目录时，将抛异常,或者结果为false
if (null != curPath) {
// 回到原来的目录
ftpClient.changeWorkingDirectory(curPath);
}
return rst;
} catch (Exception e) {
return false;
}
}
}
/**
* 判断ftp上的指定路径是否是文件
*
* @param ftpClient FTPClient
* @param remote 远端的ftp路径
* @return ftp上的指定路径是否是文件
*/
public static boolean isFile(FTPClient ftpClient, String remote) {
if (null == ftpClient || StringUtils.isBlank(remote)) return false;
remote = encoding(remote);
if (!exists(ftpClient, remote)) return false;
//尝试使用MLST命令,该命令在老的ftp上不支持
FTPFile ftpFile;
try {
ftpFile = ftpClient.mlistFile(remote);
} catch (IOException e) {
ftpFile = null;
}
if (null != ftpFile) {
//该FTP支持MLST命令
return ftpFile.isFile();
} else {
//该FTP不支持MLST命令,换用另一种兼容方式
try {
String curPath = ftpClient.printWorkingDirectory();
boolean rst = ftpClient.changeWorkingDirectory(remote); //不是目录时，将抛异常,或者结果为false
if (null != curPath) {
// 回到原来的目录
ftpClient.changeWorkingDirectory(curPath);
}
return !rst;
} catch (Exception e) {
return true;
}
}
}
/**
* 从FTP下载文件或目录,本地已经存在的同名文件会被覆盖;
* 实现功能和以下的本地复制命令一致(但local不存在时会自动创建):
* cp -rf remote local
*
* @param ftpClient FTPClient
* @param remote 远端的ftp文件或ftp目录
* @param local 本地的目录
* @return 是否成功下载
*/
public static boolean downloadFtpFiles(FTPClient ftpClient, String remote, String local) {
if (null == ftpClient || StringUtils.isBlank(remote) || StringUtils.isBlank(local)) {
logger.error("下载ftp文件函数的入参:FTPClient,远端的ftp文件路径,或者本地文件路径为空.");
return false;
}
//去除末尾的/符号
remote = trimEndSeparator(remote);
local = trimEndSeparator(local);
return _downloadFtpFiles(ftpClient, remote, local);
}
private static boolean _downloadFtpFiles(FTPClient ftpClient, String remote, String local) {
try {
String remoteEncoding = encoding(remote);
String remoteName = remoteEncoding.substring(remoteEncoding.lastIndexOf(FTP_PATH_SEPARATOR) + 1);
String localDecoding = local + File.separator + decoding(remoteName);
File localFile = new File(localDecoding);
boolean success = true;
if (isFile(ftpClient, remoteEncoding)) {
//待下载的为文件:直接下载
if (localFile.exists()) {
success = localFile.delete();
}
if (!localFile.getParentFile().exists()) {
boolean oneTry = localFile.getParentFile().mkdirs();
//多线程时尝试建立目录可能会失败,直接检查结果即可
success &= oneTry || localFile.getParentFile().exists();
}
success &= localFile.createNewFile();
if (!success) {
logger.error("尝试删除本地同名文件并建立新文件时出错.");
return false;
}
// 输出到文件
OutputStream fos = new FileOutputStream(localFile);
// 下载文件
ftpClient.retrieveFile(remoteEncoding, fos);
fos.close();
} else if (isDirecotory(ftpClient, remoteEncoding)) {
//待下载的为文件夹:递归下载目录内的子文件/目录
ftpClient.changeWorkingDirectory(remoteEncoding);
if (!localFile.exists()) {
success = localFile.mkdirs();
//多线程下可能会失败,所以要检查结果
if (!success && !localFile.exists()) {
logger.error("尝试建立新文件夹时出错.");
return false;
}
}
FTPFile[] files = ftpClient.listFiles();
for (FTPFile file : files) {
if (".".equals(file.getName()) || "..".equals(file.getName())) {
continue;
}
String remoteSonEncoding = remoteEncoding + FTP_PATH_SEPARATOR + encoding(file.getName());
boolean rst = _downloadFtpFiles(ftpClient, remoteSonEncoding, localDecoding);
if (!rst) return false;
}
} else {
logger.error("待下载的ftp文件的类型识别错误.");
return false;
}
return true;
} catch (Exception e) {
logger.error("执行下载ftp文件的过程中,出现异常.", e);
return false;
}
}
/**
* 上传本地的文件或目录到FTP,如果ftp已经存在同名文件则进行覆盖;
* 实现功能和以下的本地复制命令一致(但remote不存在时会自动创建):
* cp -rf local remote
*
* @param ftpClient FTPClient
* @param local 本地的文件或目录
* @param remote 远端的ftp目录
* @return 是否成功上传
*/
public static boolean uploadFtpFiles(FTPClient ftpClient, String local, String remote) {
if (null == ftpClient || StringUtils.isBlank(remote) || StringUtils.isBlank(local)) {
logger.error("上传ftp文件函数的入参:FTPClient,远端的ftp文件路径,或者本地文件路径为空.");
return false;
}
//去除末尾的/符号
remote = trimEndSeparator(remote);
return _uploadFtpFiles(ftpClient, local, remote);
}
private static boolean _uploadFtpFiles(FTPClient ftpClient, String local, String remote) {
try {
File localFile = new File(local);
String remoteEncoding = encoding(remote);
String fileNameEncoding = encoding(localFile.getName());
String remoteSonEncoding = remoteEncoding + FTP_PATH_SEPARATOR + fileNameEncoding;
boolean success;
if (localFile.isFile()) {
//待上传的为文件:直接上传
success = FTPUtil.mkdirs(ftpClient, remoteEncoding);
//多线程下可能创建目录失败,故需要直接检查结果
if(!success && !FTPUtil.exists(ftpClient, remoteEncoding)){
logger.error("创建ftp上所需父目录:{}时失败.",remoteEncoding);
return false;
}
FileInputStream fis = new FileInputStream(localFile);
ftpClient.setFileType(FTPClient.BINARY_FILE_TYPE);
ftpClient.changeWorkingDirectory(remoteEncoding);
ftpClient.storeFile(fileNameEncoding, fis);
fis.close();
} else if (localFile.isDirectory()) {
//待上传的为文件夹:先创建目录,然后递归上传目录内的子文件/目录
success = FTPUtil.mkdirs(ftpClient, remoteSonEncoding);
//多线程下可能创建目录失败,故需要直接检查结果
if(!success && !FTPUtil.exists(ftpClient, remoteSonEncoding)){
logger.error("创建ftp上所需目录:{}时失败.", remoteSonEncoding);
return false;
}
File[] files = localFile.listFiles();
if (null == files) {
logger.error("解析本地文件时出错.");
return false;
}
for (File f : files) {
boolean rst = _uploadFtpFiles(ftpClient, f.getCanonicalPath(), remoteSonEncoding);
if (!rst) return false;
}
} else {
logger.error("待上传的文件的类型识别错误.");
return false;
}
return true;
} catch (Exception e) {
logger.error("执行上传ftp文件的过程中,出现异常.", e);
return false;
}
}
/**
* 递归创建ftp上的目录,如果已经存在则不创建
*
* @param ftpClient FTPClient
* @param remote 远端的ftp目录
* @return 是否创建目录成功
*/
public static boolean mkdirs(FTPClient ftpClient, String remote) {
try {
if (null == ftpClient || StringUtils.isBlank(remote)) {
logger.error("创建ftp文件函数的入参:FTPClient或者远端的ftp文件路径为空.");
return false;
}
String remoteEncoding = encoding(remote);
if (exists(ftpClient, remoteEncoding)) {
return true;
}
boolean success;
StringBuilder sb = new StringBuilder();
String[] nodes = remoteEncoding.split(FTP_PATH_SEPARATOR);
for (String node : nodes) {
if (null == node || "".equals(node)) continue;
sb.append(FTP_PATH_SEPARATOR).append(node);
String remoteParentEncoding = encoding(sb.toString());
if (!exists(ftpClient, remoteParentEncoding)) {
success = ftpClient.makeDirectory(remoteParentEncoding);
//多线程情况下可能会失败,所以需要直接检查结果
if (!success && !exists(ftpClient, remoteParentEncoding)) {
logger.error("创建目录:{}时失败.", sb.toString());
return false;
}
}
}
return true;
} catch (Exception e) {
logger.error("递归创建ftp目录的过程中,出现异常.", e);
return false;
}
}
/**
* 递归删除ftp上的文件或目录
*
* @param ftpClient FTPClient
* @param remote 远端的ftp文件或ftp目录
* @return 是否删除成功
*/
public static boolean removeFiles(FTPClient ftpClient, String remote) {
if (null == ftpClient || StringUtils.isBlank(remote)) {
logger.error("删除ftp文件函数的入参:FTPClient或者远端的ftp文件路径为空.");
return false;
}
//去除末尾的/符号
remote = trimEndSeparator(remote);
if (!exists(ftpClient, remote)) {
logger.error("待删除的文件不存在,无法删除.");
return false;
}
return _removeFiles(ftpClient, remote);
}
private static boolean _removeFiles(FTPClient ftpClient, String remote) {
try {
String remoteEncoding = encoding(remote);
ftpClient.changeWorkingDirectory(remoteEncoding);
boolean success;
if (isFile(ftpClient, remoteEncoding)) {
//待删除的为文件:直接删除
success = ftpClient.deleteFile(remoteEncoding);
} else if (isDirecotory(ftpClient, remoteEncoding)) {
//待删除的为文件夹:先递归删除目录内的子文件/目录,然后删除本目录
FTPFile[] files = ftpClient.listFiles();
for (FTPFile file : files) {
if (".".equals(file.getName()) || "..".equals(file.getName())) {
continue;
}
String remoteSonEncoding = remoteEncoding + FTP_PATH_SEPARATOR + encoding(file.getName());
boolean rst = _removeFiles(ftpClient, remoteSonEncoding);
if (!rst) return false;
}
success = ftpClient.removeDirectory(remoteEncoding);
} else {
logger.error("待删除的ftp文件的类型识别错误.");
return false;
}
return success;
} catch (Exception e) {
logger.error("执行删除ftp文件的过程中,出现异常.", e);
return false;
}
}
/**
* 获取给出的远端的ftp路径的最底层的文件夹集合的路径,以方便按照最底层文件夹级别进行多线程下载
*
* @param ftpClient FTPClient
* @param remote 远端的ftp路径
* @return 最底层的文件夹路径的集合
*/
public static List<String> getBottomDirs(FTPClient ftpClient, String remote) {
if (null == ftpClient || StringUtils.isBlank(remote)) {
logger.error("获取ftp底层文件夹路径的函数的入参:FTPClient或者远端的ftp文件路径为空.");
return new ArrayList<String>();
}
//去除末尾的/符号
remote = trimEndSeparator(remote);
return _getBottomDirs(ftpClient, remote);
}
private static List<String> _getBottomDirs(FTPClient ftpClient, String remote) {
List<String> result = new ArrayList<String>();
try {
String remoteEncoding = encoding(remote);
if (!FTPUtil.isDirecotory(ftpClient, remoteEncoding)) return result;
ftpClient.changeWorkingDirectory(remoteEncoding);
FTPFile[] files = ftpClient.listFiles();
for (FTPFile file : files) {
if (".".equals(file.getName()) || "..".equals(file.getName())) {
continue;
}
String remoteSonEncoding = remoteEncoding + FTP_PATH_SEPARATOR + encoding(file.getName());
List<String> sonRst = _getBottomDirs(ftpClient, remoteSonEncoding);
if (!sonRst.isEmpty()) {
//合并子文件夹结果
for (String rst : sonRst) {
result.add(rst);
}
}
}
if (result.isEmpty()) {
//本身就是最底层文件夹
result.add(remoteEncoding);
}
return result;
} catch (Exception e) {
logger.error("获取ftp底层文件夹路径的集合的过程中,出现异常.");
return new ArrayList<String>();
}
}
/**
* 获取给出的远端的ftp路径的最底层的文件和文件夹集合的路径,以方便按照最底层文件和文件夹级别进行多线程下载
*
* @param ftpClient FTPClient
* @param remote 远端的ftp路径
* @return 最底层的文件和文件夹路径的集合
*/
public static List<String> getBottomFiles(FTPClient ftpClient, String remote) {
if (null == ftpClient || StringUtils.isBlank(remote)) {
logger.error("获取ftp底层文件和文件夹路径的函数的入参:FTPClient或者远端的ftp文件路径为空.");
return new ArrayList<String>();
}
if (!FTPUtil.exists(ftpClient, remote)) {
logger.error("ftp文件路径不存在.");
return new ArrayList<String>();
}
//去除末尾的/符号
remote = trimEndSeparator(remote);
return _getBottomFiles(ftpClient, remote);
}
private static List<String> _getBottomFiles(FTPClient ftpClient, String remote) {
List<String> result = new ArrayList<String>();
try {
String remoteEncoding = encoding(remote);
if (FTPUtil.isFile(ftpClient, remoteEncoding)) {
//已经是最底层的文件
result.add(remoteEncoding);
return result;
}
ftpClient.changeWorkingDirectory(remoteEncoding);
FTPFile[] files = ftpClient.listFiles();
for (FTPFile file : files) {
if (".".equals(file.getName()) || "..".equals(file.getName())) {
continue;
}
String remoteSonEncoding = remoteEncoding + FTP_PATH_SEPARATOR + encoding(file.getName());
List<String> sonRst = _getBottomFiles(ftpClient, remoteSonEncoding);
if (!sonRst.isEmpty()) {
//合并子文件夹结果
for (String rst : sonRst) {
result.add(rst);
}
}
}
if (result.isEmpty()) {
//本身就是最底层文件夹(该文件夹下无子文件或子文件夹)
result.add(remoteEncoding);
}
return result;
} catch (Exception e) {
logger.error("获取ftp底层文件和文件夹路径的集合的过程中,出现异常.");
return new ArrayList<String>();
}
}
private static String trimEndSeparator(String oriPath) {
if (null == oriPath) return null;
while ((oriPath.endsWith(File.separator) || oriPath.endsWith(FTP_PATH_SEPARATOR)) && oriPath.length() > 1) {
oriPath = oriPath.substring(0, oriPath.length() - 1);
}
return oriPath;
}
public static void main(String[] args) {
}
/**
* 自定义实现ftp连接池
*/
private class FTPClientPool implements ObjectPool<FTPClient> {
private static final int DEFAULT_POOL_SIZE = 10;
private static final int WAIT_TIME_OUT = 1;
private String error;
private BlockingQueue<FTPClient> pool;
private FTPClientFactory factory;
public FTPClientPool(FTPClientFactory factory) throws Exception {
this(DEFAULT_POOL_SIZE, factory);
}
/**
* 创建并初始化ftp连接池
*
* @param poolSize 连接池大小
* @param factory factory实例
* @throws Exception 因为创建并添加ftp实例失败,导致初始化失败
*/
public FTPClientPool(int poolSize, FTPClientFactory factory) throws Exception {
this.factory = factory;
this.pool = new ArrayBlockingQueue<FTPClient>(poolSize);
//初始化时预分配所有的ftp实例
for (int i = 0; i < poolSize; i++) {
addOneFtp(); //可能导致异常
}
}
/**
* 为ftp连接池创建并添加一个新的ftp实例
*
* @throws Exception 创建ftp实例失败
*/
private void addOneFtp() throws Exception {
//使用阻塞等待的方式添加,是因为BlockingQueue的读和写共用一个锁;写有可能被读所阻塞
pool.offer(factory.makeObject(), WAIT_TIME_OUT, TimeUnit.SECONDS);
}
/**
* 获取(占用)ftp连接池的一个实例
*
* @return ftp实例
* @throws Exception 获取的ftp实例无效,尝试重新创建时失败
*/
@Override
public FTPClient borrowObject() throws Exception {
FTPClient client;
client = pool.take();
if (client == null) {
client = factory.makeObject();
addOneFtp(); //创建失败时抛出异常
} else if (!factory.validateObject(client)) {
invalidateObject(client);
client = factory.makeObject();
addOneFtp(); //创建失败时抛出异常
}
return client;
}
/**
* 归还(释放)ftp连接池的一个实例
*
* @param client ftp实例
* @throws Exception 归还失败时,尝试重新创建一个ftp实例补充到连接池里却失败
*/
@Override
public void returnObject(FTPClient client) throws Exception {
if (null == client) {
throw new NullPointerException("FTPClient为null.");
}
try {
if (!pool.offer(client, WAIT_TIME_OUT, TimeUnit.SECONDS)) {
//因为等待超时,导致将ftp实例放回连接池不成功
error = "因为等待超时,导致将ftp实例放回连接池不成功";
logger.error(error);
factory.destroyObject(client);
addOneFtp(); //添加不成功会抛出异常
}
} catch (InterruptedException e) {
//因为被人为中断,导致将ftp实例放回连接池不成功
error = "因为被人为中断,导致将ftp实例放回连接池不成功";
logger.error(error, e);
factory.destroyObject(client);
addOneFtp(); //添加不成功会抛出异常
}
}
/**
* 移除无效的对象(FTP客户端)
*
* @param client ftp实例
*/
@Override
public void invalidateObject(FTPClient client) {
if (null != client) {
pool.remove(client);
}
}
/**
* 关闭连接池并释放资源
*/
@Override
public void close() {
while (pool.iterator().hasNext()) {
FTPClient client;
try {
client = pool.take();
} catch (InterruptedException e) {
logger.error("从线程池中获取ftp实例时被中断.", e);
continue;
}
factory.destroyObject(client);
}
}
/**
* 添加一个ftp实例(不支持)
*
* @throws UnsupportedOperationException
*/
@Override
public void addObject() throws UnsupportedOperationException {
throw new UnsupportedOperationException();
}
/**
* 获取空闲链接数(不支持)
*
* @return 空闲链接数
* @throws UnsupportedOperationException
*/
@Override
public int getNumIdle() throws UnsupportedOperationException {
throw new UnsupportedOperationException();
}
/**
* 获取正在被使用的链接数(不支持)
*
* @return 正在被使用的链接数
* @throws UnsupportedOperationException
*/
@Override
public int getNumActive() throws UnsupportedOperationException {
throw new UnsupportedOperationException();
}
/**
* 清除空闲ftp实例并释放资源.(不支持该方法)
*
* @throws UnsupportedOperationException
*/
@Override
public void clear() throws UnsupportedOperationException {
throw new UnsupportedOperationException();
}
/**
* 替换factory实例(不支持该方法)
*
* @param poolableObjectFactory factory实例
* @throws UnsupportedOperationException
*/
@Override
public void setFactory(PoolableObjectFactory<FTPClient> poolableObjectFactory) throws UnsupportedOperationException {
throw new UnsupportedOperationException();
}
}
/**
* 连接池工厂类
*/
private class FTPClientFactory implements PoolableObjectFactory<FTPClient> {
private String _ftpUrl;
/**
* 使用ftp的url,初始化factory类
*
* @param ftpUrl ftp的Url,格式:ftp://user:pass@xx.xx.xx.xx:port
* 其中,ftp://前缀,用户名密码,端口号都不是必须的
* 若没有用户名密码则匿名登录
* 没有端口号则使用FTP默认端口号
*/
public FTPClientFactory(String ftpUrl) {
this._ftpUrl = ftpUrl;
}
/**
* 获取一个有效的ftp实例
*
* @return 一个有效的ftp实例
* @throws Exception 无法创建ftp实例
*/
@Override
public FTPClient makeObject() throws Exception {
String username;
String password;
int port;
String error;
//对内部的ftpUrl复制后再操作
String ftpUrl = this._ftpUrl;
if (StringUtils.isBlank(ftpUrl)) {
error = "ftp的URL为空.";
logger.error(error);
throw new NullPointerException(error);
}
//去掉ftp前缀
if (StringUtils.startsWithIgnoreCase(ftpUrl, FTP_PREFIX)) {
ftpUrl = ftpUrl.substring(FTP_PREFIX.length());
}
//去掉path
if (ftpUrl.contains(FTP_PATH_SEPARATOR)) {
ftpUrl = ftpUrl.substring(0, ftpUrl.indexOf(FTP_PATH_SEPARATOR));
}
//获取用户名密码
int hostIndex = ftpUrl.indexOf(FTP_HOST_SEPARATOR);
if (hostIndex >= 0) {
//ftpUrl中包含用户名密码,需要提取
int passIndex = ftpUrl.indexOf(FTP_PASSWORD_SEPARATOR);
if (passIndex > 0 && passIndex < hostIndex) {
String account = ftpUrl.substring(0, hostIndex);
ftpUrl = ftpUrl.substring(hostIndex + 1);
username = account.substring(0, passIndex);
password = account.substring(passIndex + 1);
} else {
error = "ftp的URL格式错误,未提取到登录的用户名和密码.";
logger.error(error);
throw new IllegalArgumentException(error);
}
} else {
//ftpUrl不包含用户名密码,使用匿名登录
username = FTP_ANONYMOUS_NAME;
password = FTP_ANONYMOUS_PASSWORD;
}
//获取端口
int portIndex = ftpUrl.indexOf(FTP_PORT_SEPARATOR);
if (portIndex >= 0) {
//ftpUrl中指定了端口号
port = Integer.parseInt(ftpUrl.substring(portIndex + 1));
} else {
//ftpUrl中未指定端口号,使用默认端口
port = FTP_DEFAULT_PORT;
}
boolean flag;
FTPClient ftpClient = new FTPClient();
try {
ftpClient.connect(ftpUrl, port);
flag = ftpClient.login(username, password);
} catch (IOException e) {
error = "建立FTP连接或者登录出现错误.";
logger.error(error, e);
throw e;
}
if (flag) {
ftpClient.setControlKeepAliveTimeout(FTP_KEEP_ALIVE_TIMEOUT);
ftpClient.setBufferSize(FTP_BUFFER_SIZE);
//尝试进入被动模式
ftpClient.enterLocalPassiveMode();
// 开启服务器对UTF-8的支持，如果服务器支持就用UTF-8编码，否则就使用GBK.
try {
if (!FTPReply.isPositiveCompletion(ftpClient.sendCommand("OPTS UTF8", "ON"))) {
localCharset = ENCODING_GBK;
}
} catch (IOException e) {
localCharset = ENCODING_GBK;
}
return ftpClient;
} else {
error = "登录FTP失败.";
logger.error(error);
throw new RuntimeException(error);
}
}
/**
* 尝试关闭ftp实例
*
* @param ftpClient ftp实例
*/
@Override
public void destroyObject(FTPClient ftpClient) {
try {
if (ftpClient != null && ftpClient.isConnected()) {
ftpClient.logout();
}
} catch (Exception e) {
logger.error("ftp client logout failed...", e);
} finally {
if (ftpClient != null) {
try {
ftpClient.disconnect();
} catch (IOException e) {
logger.error("ftp client disconnect failed...", e);
}
}
}
}
/**
* 验证ftp实例是否可用
*
* @param ftpClient ftp实例
* @return 是否可用
*/
@Override
public boolean validateObject(FTPClient ftpClient) {
try {
if (null != ftpClient && ftpClient.isConnected()) {
return ftpClient.sendNoOp();
}
} catch (Exception e) {
logger.error("Failed to validate client: {}", e);
}
return false;
}
/**
* 激活一个实例(ftp连接池不支持该方法)
*
* @param obj ftp实例
* @throws Exception
*/
@Override
public void activateObject(FTPClient obj) throws Exception {
//Do nothing
throw new UnsupportedOperationException();
}
/**
* 反激活一个实例(ftp连接池不支持该方法)
*
* @param obj ftp实例
* @throws Exception
*/
@Override
public void passivateObject(FTPClient obj) throws Exception {
//Do nothing
throw new UnsupportedOperationException();
}
}
}
文件操作
FileUtil.java
package com;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.net.ftp.FTPClient;
import org.apache.commons.net.ftp.FTPFile;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
/**
* Created by zhy.li on 16/9/22.
*/
public class FileUtil {
private static Logger logger = LoggerFactory.getLogger(FileUtil.class);
private FileUtil() {
}
/**
* 如果文件或目录存在,则删除它们
*
* @param file 文件句柄
* @return 是否删除成功
*/
public static boolean removeFiles(File file) {
try {
if (null == file) {
logger.error("参数错误,待删除的文件句柄为null.");
return false;
}
if (!file.exists()) {
logger.info("文件不存在,无法删除.");
return false;
}
boolean success;
if (file.isFile()) {
//待删除的为文件:直接删除
success = file.delete();
} else if (file.isDirectory()) {
//待删除的为文件夹:先递归删除目录内的子文件/目录,然后删除本目录
File[] files = file.listFiles();
if (null == files) {
logger.error("在获取删除目录内的子文件/目录信息时出现错误.");
return false;
}
for (File sonFile : files) {
if (".".equals(sonFile.getName()) || "..".equals(sonFile.getName())) {
continue;
}
boolean rst = removeFiles(sonFile);
if (!rst) return false;
}
success = file.delete();
} else {
logger.error("待删除文件的类型识别错误.");
return false;
}
return success;
} catch (Exception e) {
logger.error("执行删除文件的过程中,出现异常.", e);
return false;
}
}
/**
* 获取给出的路径的最底层的文件夹路径的集合,以方便按照最底层文件夹级别进行多线程处理
*
* @param file 文件夹路径
* @return 最底层的文件夹路径的集合
*/
public static List<String> getBottomDirs(File file) {
List<String> result = new ArrayList<String>();
if (null == file) {
logger.error("参数错误,文件句柄为null.");
return result;
}
try {
if (!file.isDirectory()) return result;
File[] files = file.listFiles();
if (null == files) return result;
for (File sonFile : files) {
if (".".equals(sonFile.getName()) || "..".equals(sonFile.getName())) {
continue;
}
List<String> sonRst = getBottomDirs(sonFile);
if (!sonRst.isEmpty()) {
//合并子文件夹结果
for (String rst : sonRst) {
result.add(rst);
}
}
}
if (result.isEmpty()) {
//本身就是最底层文件夹
result.add(file.getCanonicalPath());
}
return result;
} catch (Exception e) {
logger.error("获取最底层文件夹路径的集合的过程中,出现异常.");
return new ArrayList<String>();
}
}
/**
* 获取给出的路径的最底层的文件和文件夹路径的集合,以方便按照最底层文件和文件夹级别进行多线程处理
*
* @param file 文件夹路径
* @return 最底层的文件和文件夹路径的集合
*/
public static List<String> getBottomFiles(File file) {
List<String> result = new ArrayList<String>();
if (null == file) {
logger.error("参数错误,文件句柄为null.");
return result;
}
try {
if (!file.exists()) return result;
if (file.isFile()) {
//已经是最底层文件
result.add(file.getCanonicalPath());
return result;
}
File[] files = file.listFiles();
if (null == files) return result;
for (File sonFile : files) {
if (".".equals(sonFile.getName()) || "..".equals(sonFile.getName())) {
continue;
}
List<String> sonRst = getBottomFiles(sonFile);
if (!sonRst.isEmpty()) {
//合并子文件夹结果
for (String rst : sonRst) {
result.add(rst);
}
}
}
if (result.isEmpty()) {
//本身就是最底层文件夹(该文件夹已经没有子文件和子文件夹)
result.add(file.getCanonicalPath());
}
return result;
} catch (Exception e) {
logger.error("获取最底层文件和文件夹路径的集合的过程中,出现异常.");
return new ArrayList<String>();
}
}
/**
* 获取输入的文件和文件夹集合的上一层目录,注意:不验证输入的路径是否存在
*
* @param lowerFiles 某一层的文件和文件夹路径集合
* @return 上一层目录路径的集合
*/
public static List<String> getUpperDirs(List<String> lowerFiles, String separator) {
List<String> result = new ArrayList<String>();
if (null == lowerFiles || lowerFiles.isEmpty()) return result;
for (String lowerFile : lowerFiles) {
//去除末尾的/符号
lowerFile = trimEndSeparator(lowerFile);
String upperDir = lowerFile.substring(0, lowerFile.lastIndexOf(separator));
if (!result.contains(upperDir) && !"".equals(upperDir)) {
result.add(upperDir);
}
}
return result;
}
private static String trimEndSeparator(String oriPath) {
if (null == oriPath) return null;
while ((oriPath.endsWith(File.separator) || oriPath.endsWith(FTPUtil.FTP_PATH_SEPARATOR)) && oriPath.length() > 1) {
oriPath = oriPath.substring(0, oriPath.length() - 1);
}
return oriPath;
}
public static void main(String[] args) {
}
}
Json操作
jsonUtil.java
package com;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.serializer.SerializeConfig;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.google.common.collect.Sets;
import com.qunar.base.qunit.ex.constants.IgnoreDate;
import com.qunar.base.qunit.fastjson.QunitDoubleSerializer;
import com.qunar.base.qunit.response.Response;
import com.qunar.base.qunit.util.PropertyUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
public class JsonUtil {
private static Logger logger = LoggerFactory.getLogger(JsonUtil.class);
private static final String KEY_PREFIX_SEPERATOR = ".";
private static final String AT_MAGIC_FLAG = "__at_magic_flag_411411__";
// Json to Map
@Deprecated
private static List<Map<String, Object>> json2List(Object json) {
JSONArray jsonArr = (JSONArray) json;
List<Map<String, Object>> arrList = new ArrayList<Map<String, Object>>();
for (int i = 0; i < jsonArr.size(); ++i) {
arrList.add(strJson2Map(jsonArr.getString(i)));
}
return arrList;
}
@Deprecated
public static Map<String, Object> strJson2Map(String json) {
JSONObject jsonObject = JSONObject.parseObject(json);
Map<String, Object> resMap = new HashMap<String, Object>();
Iterator<Map.Entry<String, Object>> it = jsonObject.entrySet().iterator();
while (it.hasNext()) {
Map.Entry<String, Object> param = (Map.Entry<String, Object>) it.next();
if (param.getValue() instanceof JSONObject) {
resMap.put(param.getKey(), strJson2Map(param.getValue().toString()));
} else if (param.getValue() instanceof JSONArray) {
resMap.put(param.getKey(), json2List(param.getValue()));
} else {
resMap.put(param.getKey(), JSONObject.toJSONString(param.getValue(), SerializerFeature.WriteClassName));
}
}
return resMap;
}
public static Map<String, Object> json2Map(String json) {
logger.debug("开始将json串转换为Map.");
Map<String, Object> rst = new HashMap<String, Object>();
Object source;
if (StringUtils.isBlank(json)) {
logger.debug("json串为空串,返回空结果");
return rst;
}
//预处理,防止@形式的键值被转换为对象,在转换为map时再替换回来
json = json.replaceAll("@", AT_MAGIC_FLAG);
try {
source = JSON.parse(json);
} catch (Exception e) {
//json不是Json字符串
logger.debug("解析成json格式失败,返回null.待解析的Json内容为:{}", json);
return null;
}
logger.debug("开始循环递归获取基本值的key前缀Map.");
getKeyPrefixMap(null, rst, source);
logger.debug("json串已经转换为map.");
return rst;
}
private static void getKeyPrefixMap(String keyPrefix, Map<String, Object> result, Object object) {
if (null == result) {
String error = "递归获取入key前缀Map时,传入的保存结果的Map为null.";
logger.error(error);
throw new NullPointerException(error);
}
if (object instanceof JSONObject) {
//object为Json串,循环调用该Json串的所有键值
JSONObject jsonObject = (JSONObject) object;
String nextKeyPrefix = keyPrefix == null ? "" : keyPrefix + KEY_PREFIX_SEPERATOR;
for (String key : jsonObject.keySet()) {
getKeyPrefixMap(nextKeyPrefix + key, result, jsonObject.get(key));
}
} else if (object instanceof JSONArray) {
//object为Json数组,循环调用该Json串的所有
JSONArray jsonArray = (JSONArray) object;
String nextKeyPrefix = keyPrefix == null ? "" : keyPrefix + KEY_PREFIX_SEPERATOR;
for (int i = 0; i < jsonArray.size(); i++) {
getKeyPrefixMap(nextKeyPrefix + "[" + i + "]", result, jsonArray.get(i));
}
} else {
//object为基本值,得到一个结果;首先进行@键值的还原
if (object instanceof String) {
object = ((String) object).replaceAll(AT_MAGIC_FLAG, "@");
}
result.put(keyPrefix.replaceAll(AT_MAGIC_FLAG, "@"), object);
}
}
public static Set<String> diffMap(Map<String, Object> source, Map<String, Object> target, String dateIgnore) {
Set<String> result = Sets.newHashSet();
//IgnoreDate type = IgnoreDate.getIgnoreType(ignoreType);
DateProcessor dateProcessor = new DateProcessor();
// todo: fixme:该变量可以下放到对类型type=VALUE处理时
//List<String> datePattern = dateProcessor.getDatePattern(ignoreType);
for (String key : source.keySet()) {
if (target.containsKey(key)) {
//target存在这个key,进行比较值
Object s = source.get(key);
Object t = target.get(key);
// 注意:生成ignore和做全量assert均使用该函数,此处目的为将需要日期忽略的key加入忽略列表.
// 会造成在assert时被直接认为是不同的结果,之所以现在不会出问题,是因为assert比较前已经剔除了忽略的key
if (t != null && !IgnoreDate.NULL.equals(IgnoreDate.getIgnoreType(dateIgnore))) {
if (dateProcessor.isDate(dateIgnore, t.toString())) {
result.add(key);
continue;
}
} /*else if (t != null && type.equals(IgnoreDate.VALUE)) {
for (String pattern : datePattern) {
if (dateProcessor.fuzzyMatch(key, pattern) || dateProcessor.isDate(ignoreType,
t.toString())) {
result.add(key);
continue;
}
}
}*/
if (!isObjectEqual(s, t)) {
result.add(key);
}
} else {
//target不存在这个key
result.add(key);
}
}
for (String key : target.keySet()) {
if (!source.containsKey(key)) {
//else: 会在上面的循环中进行处理
result.add(key);
}
}
return result;
}
public static Set<String> diffMapWithDisorderArray(Map<String, Object> source, Map<String, Object> target, String dateIgnore) {
Set<String> result = Sets.newHashSet();
Set<String> equalRst = Sets.newHashSet(); //空间换时间,记录相等的结果用于全量比较时提效
//IgnoreDate type = IgnoreDate.getIgnoreType(ignoreType);
DateProcessor dateProcessor = new DateProcessor();
// todo: fixme:该变量可以下放到对类型type=VALUE处理时
//List<String> datePattern = dateProcessor.getDatePattern(ignoreType);
for (String key : source.keySet()) {
if (target.containsKey(key)) {
//target存在这个key,进行比较值
Object s = source.get(key);
Object t = target.get(key);
// 注意:生成ignore和做全量assert均使用该函数,此处目的为将需要日期忽略的key加入忽略列表.
// 会造成在assert时被直接认为是不同的结果,之所以现在不会出问题,是因为assert比较前已经剔除了忽略的key
if (t != null && !IgnoreDate.NULL.equals(IgnoreDate.getIgnoreType(dateIgnore))) {
if (dateProcessor.isDate(dateIgnore, t.toString())) {
result.add(key);
continue;
}
} /*else if (t != null && type.equals(IgnoreDate.VALUE)) {
for (String pattern : datePattern) {
if (dateProcessor.fuzzyMatch(key, pattern) || dateProcessor.isDate(ignoreType,
t.toString())) {
result.add(key);
continue;
}
}
}*/
// 数组无序比较之前先同位置比较,在数组基本有序时可以提高效率
if (isObjectEqual(s, t)) {
equalRst.add(key);
continue;
}
// 进行数组无序比较
if (!disorderArrayAssert("", key, source.get(key), target)) {
result.add(key);
} else {
equalRst.add(key);
}
} else {
//target不存在这个key
result.add(key);
}
}
// 全量比较,现在只需要找出在target中有,但在source中不存在的那些key
for (String key : target.keySet()) {
if (!source.containsKey(key) && !equalRst.contains(key)) {
result.add(key);
}
}
return result;
}
private static boolean isObjectEqual(Object s, Object t) {
if (null == s && null == t) {
//2个key都为null
return true;
}
if ((null == s) || (null == t)) {
//只有1个key为null,另1个不为null
return false;
}
//因为sonar检查而注释
// if (s == t) {
// //是同一个对象
// continue;
// }
if (s.equals(t)) {
//两个对象"相等"
return true;
}
if (s.getClass() == t.getClass() && s.toString().equals(t.toString())) {
//两个对象的类型相同,且转换为字符串后比较相同
return true;
}
//非以上情况,视为key的类型不同或者值不同
return false;
}
private static boolean disorderArrayAssert(String keyPrefix, String expKey, Object expValue, Map<String, Object> resultAct) {
int arrayBegin = expKey.indexOf("[");
if (arrayBegin < 0) {
//递归出口:不含有数组, 直接进行比较即可
String fullKey = (StringUtils.isBlank(keyPrefix) ? "" : keyPrefix) + expKey;
if (resultAct.containsKey(fullKey)) {
return isObjectEqual(expValue, resultAct.get(fullKey));
} else {
return false;
}
} else {
// 将最上层的数组循环进行处理, 递归调用求解
int arrayEnd = expKey.indexOf("]");
if (arrayEnd <= arrayBegin) {
logger.error("Error: 字符串格式解析错误,未找到匹配的数组下标,字符串为: {}", expKey);
return false;
}
String arrayPrefix = (StringUtils.isBlank(keyPrefix) ? "" : keyPrefix) + expKey.substring(0, arrayBegin + 1);
String arrayPostfix = expKey.substring(arrayEnd, expKey.length());
int i = 0;
while (true) {
String arrayFull = arrayPrefix + i + arrayPostfix;
if (resultAct.containsKey(arrayFull)) {
//期望中存在对应的数组原因, 递归调用进行比较
if (disorderArrayAssert(arrayPrefix + i + "]", arrayPostfix.substring(1, arrayPostfix.length()), expValue, resultAct)) {
return true;
}
} else {
// 数组越界
return false;
}
i += 1;
}
}
}
public static String response2Json(Response response) {
if (null == response) return null;
//处理body非json的情况
Object body = response.getBody();
Object bodyJson;
try {
if (body instanceof String) {
//字符串被JSON.toJSON()设定为基本类型,不再处理而是直接返回
bodyJson = JSON.parse((String) body);
response.setBody(bodyJson);
} else if (!(body instanceof JSON)) {
bodyJson = JSON.toJSON(body);
response.setBody(bodyJson);
}
} catch (Exception e) {
logger.error("尝试将Response的body转变为Json格式时出错,将直接保持body为原格式不变");
}
Boolean jsonWriteOriginalDoubleValue = Boolean.valueOf(PropertyUtils.getProperty("json_write_original_double_value", "false"));
SerializeConfig config = new SerializeConfig();
if (jsonWriteOriginalDoubleValue) {
config.setAsmEnable(false);
config.put(Double.class, QunitDoubleSerializer.INSTANCE);
}
return JSON.toJSONString(response, config, SerializerFeature.WriteMapNullValue);
}
public static void main(String[] args) {
//{"key1": "1111", "key2" : {"key21": 2121, "key22":[221, 222, "223"]}}
}
}
基于DBUnit的数据表比较
/**
* 比较两个数据表,得到不同的数据表或字段
* @param sourceTable 待比较的数据表
* @param targetTable 待比较的数据表
* @return 2个数据表不同之处,格式:1)table1(col1,col2); 表示:table1的col1和col2字段不同; 2)table1; 表示:table1的整个数据表都不同
*/
public static String generateTableDiff(ITable sourceTable, ITable targetTable) {
logger.info("开始进行数据表的比较");
if ((null == sourceTable || null == sourceTable.getTableMetaData()) && (null == targetTable || null == targetTable.getTableMetaData())) {
logger.info("2个数据表的内容均为空,返回结果:无差异");
return "";
}
if (null == sourceTable || null == sourceTable.getTableMetaData()) {
logger.info("1个数据表的内容不为空,另一个数据表的内容为空,返回结果:整个数据表均有差异");
return targetTable.getTableMetaData().getTableName() + ";";
}
if (null == targetTable || null == targetTable.getTableMetaData()) {
logger.info("1个数据表的内容不为空,另一个数据表的内容为空,返回结果:整个数据表均有差异");
return sourceTable.getTableMetaData().getTableName() + ";";
}
ITableMetaData sourceTableMetaData = sourceTable.getTableMetaData();
ITableMetaData targetTableMetaData = targetTable.getTableMetaData();
logger.info("2个数据表均不为空,2个数据表的表名为:{}和{},开始比较数据表的内容.", sourceTableMetaData.getTableName(), targetTableMetaData.getTableName());
if (!StringUtils.equals(sourceTableMetaData.getTableName(), targetTableMetaData.getTableName())) {
logger.info("2个数据表的表名不相同,返回结果:整个数据表均有差异");
return sourceTableMetaData.getTableName() + ";" + targetTableMetaData.getTableName() + ";";
}
//比较行数
if (sourceTable.getRowCount() != targetTable.getRowCount()) {
logger.info("2个数据表的行数不相同,返回结果:整个数据表均有差异");
return sourceTableMetaData.getTableName() + ";";
}
//比较列
Columns.ColumnDiff columnDiff;
Column[] sourceColumns;
Column[] targetColumns;
try {
sourceColumns = Columns.getSortedColumns(sourceTableMetaData);
targetColumns = Columns.getSortedColumns(targetTableMetaData);
columnDiff = Columns.getColumnDiff(sourceTableMetaData, targetTableMetaData);
} catch (DataSetException e) {
logger.info("读取数据表的字段结构出现异常,返回结果:整个数据表均有差异");
return sourceTableMetaData.getTableName() + ";";
}
//更新:改为仅判断期望是否有差异的字段;
// 因为数据库中为null的实际数据在保存到xml文件中时会丢失,造成在比较时,期望字段比实际数据库字段少
//if (columnDiff.hasDifference()) {
if (columnDiff.getExpected().length > 0) {
//一般case执行后不应该有列的变化;如果列有增删改时,直接返回整个数据表忽略
logger.info("2个数据表的字段结构不一致({}),返回结果:整个数据表均有差异", columnDiff.toString());
return sourceTableMetaData.getTableName() + ";";
} else if (columnDiff.getActual().length > 0) {
logger.info("实际数据表的字段比期望数据表的字段多,一般原因为该数据表有字段的实际取值为nul,因此不会记录在xml文件中导致;" +
"因为不会导致assert失败,故继续比较;内容如下:{}.", columnDiff.toString());
}
//比较列的数据类型
for (int j = 0; j < sourceColumns.length; j++) {
Column sourceColumn = sourceColumns[j];
Column targetColumn = targetColumns[j];
DataType sourceDataType = sourceColumn.getDataType();
DataType targetDataType = targetColumn.getDataType();
if (!(sourceDataType instanceof UnknownDataType) && !(targetDataType instanceof UnknownDataType) && !(sourceDataType.getClass().isInstance(targetColumn))) {
//列的数据类型均存在,且不相同;注:实际上从xml读取的字段类型总是为unknown
logger.info("2个数据表的字段结构不一致(字段{}的数据类型不同),返回结果:整个数据表均有差异", sourceColumn.getColumnName());
return sourceTableMetaData.getTableName() + ";";
}
}
//比较数据
int rowCount = sourceTable.getRowCount();
StringBuilder buf = new StringBuilder();
for (Column column : sourceColumns) {
for (int i = 0; i < rowCount; i++) {
Object sourceValue;
Object targetValue;
try {
sourceValue = sourceTable.getValue(i, column.getColumnName());
} catch (DataSetException e) {
//异常为该行数据无该列字段
sourceValue = null;
}
try {
targetValue = targetTable.getValue(i, column.getColumnName());
} catch (DataSetException e) {
//异常为该行数据无该列字段
targetValue = null;
}
if (!StringUtils.equals(sourceValue != null ? sourceValue.toString() : null, targetValue != null ? targetValue.toString() : null)) {
//发现不同数据,记录该列为diff字段,并无须再比较剩下的行
logger.info("2个数据表的字段:{}出现不同数据,进行记录并继续下个字段的检查.", column.getColumnName());
buf.append(",").append(column.getColumnName());
break;
}
}
}
String diffRst = buf.toString();
if (StringUtils.isNotBlank(diffRst)) {
logger.info("2个数据表的所有字段已经全部比较完毕,差异结果为:{}", diffRst);
return sourceTableMetaData.getTableName() + "(" + diffRst.substring(1) + ");";
}
logger.info("2个数据表的结构和数据完全一致,返回结果:无差异");
return "";
}
对特定格式字符串的处理
/**
* 将字符串形式的DB表达转换成Map方式
* @param dbStr 表示DB的字符串,格式:db1(table1(col1,col2);table2);db2(table21(co21))
* @return 转换后的Map,key为db名称,value为数据表字符串
*/
private static Map<String, String> dbStr2Map(String dbStr) {
//格式:db1(table1(col1,col2);table2);db2(table21(co21))
Map<String, String> dbMap = new HashMap<String, String>();
if (StringUtils.isBlank(dbStr)) return dbMap;
int depth = 0;
String dbName = "";
String tableStr = "";
//以char形式遍历字符串每个字符,解析字符串为Map
for (int i = 0; i < dbStr.length(); i++) {
char c = dbStr.charAt(i);
switch (c) {
case '(':
if (0 != depth) {
tableStr += c;
}
depth++;
break;
case ')':
depth--;
if (0 > depth) {
//左圆括号和右圆括号个数不一致
throw new RuntimeException("输入的DB字符串的格式错误:左圆括号和右圆括号个数不一致");
}
if (0 == depth) {
//又回到顶层,完成了遍历一个DB项,进行记录;并初始化新的遍历
if (StringUtils.isNotBlank(dbName)) {
dbMap.put(dbName, tableStr);
dbName = "";
tableStr = "";
}
} else {
tableStr += c;
}
break;
case ';':
if (0 != depth) {
tableStr += c;
}
break;
default:
if (0 == depth) {
dbName += c;
} else {
tableStr += c;
}
}
}
if (0 != depth) {
//左圆括号和右圆括号个数不一致
throw new RuntimeException("输入的DB字符串的格式错误:左圆括号和右圆括号个数不一致");
}
return dbMap;
}
/**
* 将数据表描述从字符串形式转换为Map形式
*
* @param input 字符串形式: A(a1,a2);B;C(c1)
* @return Map形式:{A=[a1,a2],B=null,C=[c1]}
*/
public static Map<String, List<String>> tablesStr2Map(String input) {
final String SEPARATOR1 = ";"; //key之间的分割符
final String SEPARATOR2 = ","; //value之间的分割符
Map<String, List<String>> result = new HashMap<String, List<String>>();
if (StringUtils.isBlank(input)) return result;
String[] tables = StringUtils.split(input, SEPARATOR1);
for (String table : tables) {
String temp = StringUtils.trim(table);
if (StringUtils.isBlank(temp)) continue;
if (temp.contains("(") && temp.endsWith(")")) {
int index = temp.indexOf("(");
String tableName = temp.substring(0, index);
if (result.containsKey(tableName) && null == result.get(tableName)) {
//此情况说明result表里已经有忽略整个数据表的表名,不应该再添加忽略的字段
continue;
}
String columnStr = temp.substring(index + 1, temp.length() - 1);
String[] columns = StringUtils.split(columnStr, SEPARATOR2);
List<String> columnList = result.get(tableName);
if (columnList == null) {
columnList = new ArrayList<String>();
result.put(tableName, columnList);
}
columnList.addAll(Arrays.asList(columns));
} else {
//if (!result.containsKey(temp)) {
result.put(temp, null);
//}
}
}
//对map的value进行trim()和去重
removeDuplicate(result);
return result;
}
/**
* 将数据表表述从Map形式转换为字符串形式
*
* @param dbMap Map形式:A={[a1,a2],B=null,C=[c1]}
* @return 字符串形式: A(a1,a2);B;C(c1)
*/
public static String tablesMap2Str(Map<String, List<String>> dbMap) {
final String SEPARATOR1 = ";"; //key之间的分割符
final String SEPARATOR2 = ","; //value之间的分割符
if (null == dbMap || dbMap.isEmpty()) return "";
//对map的value进行trim()和去重
removeDuplicate(dbMap);
String result = "";
for (String dbName : dbMap.keySet()) {
if (null != dbName) {
if (null != dbMap.get(dbName) && !dbMap.get(dbName).isEmpty()) {
result += dbName + "(";
List<String> tables = dbMap.get(dbName);
StringBuilder buf = new StringBuilder();
for (String table : tables) {
buf.append(table).append(SEPARATOR2);
}
result += buf.toString();
result = result.substring(0, result.length() - 1);
result += ")" + SEPARATOR1;
} else {
result += dbName + SEPARATOR1;
}
}
}
//去除最后的分号
if (result.endsWith(SEPARATOR1)) {
result = result.substring(0, result.length() - 1);
}
return result;
}
/**
* 1.对map的值进行trim()
* 2.对map的值进行去重
*
* @param dbMap 去重后map
*/
private static void removeDuplicate(Map<String, List<String>> dbMap) {
if (null == dbMap) return;
for (String key : dbMap.keySet()) {
List<String> value = dbMap.get(key);
if (null == value) continue;
Set<String> setValue = new HashSet<String>();
for (int i = 0; i < value.size(); i++) {
String e = value.get(i);
//先做简化处理
if (null != e) {
e = e.replaceAll("\n", "").replaceAll("\r", "").replaceAll("\t", "").trim();
}
if (setValue.contains(e)) {
//该元素已经存在
value.remove(i);
i--;
} else {
//该元素还不存在
setValue.add(e);
value.set(i, e);
}
}
}
}
扫一扫
1493
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
