文章目录
Stream
java8特性,通过将集合转换为这么一种叫做 “流” 的元素序列,通过声明性方式,能够对集合中的每个元素进行一系列并行或串行的流水线操作。
中间操作符:
map(mapToInt,mapToLong,mapToDouble) 转换操作符,把比如A->B,这里默认提供了转int,long,double的操作符。
flatmap(flatmapToInt,flatmapToLong,flatmapToDouble) 拍平操作比如把 int[]{2,3,4} 拍平 变成 2,3,4 也就是从原来的一个数据变成了3个数据,这里默认提供了拍平成int,long,double的操作符。
limit 限流操作,比如数据流中有10个 我只要出前3个就可以使用。
distint 去重操作,对重复元素去重,底层使用了equals方法。
filter 过滤操作,把不想要的数据过滤。
peek 挑出操作,如果想对数据进行某些操作,如:读取、编辑修改等。
skip 跳过操作,跳过某些元素。
sorted(unordered) 排序操作,对元素排序,前提是实现Comparable接口,当然也可以自定义比较器。
终止操作符:
collect 收集操作,将所有数据收集起来,这个操作非常重要,官方的提供的Collectors 提供了非常多收集器,可以说Stream 的核心在于Collectors。
count 统计操作,统计最终的数据个数。
findFirst、findAny 查找操作,查找第一个、查找任何一个 返回的类型为Optional。
noneMatch、allMatch、anyMatch 匹配操作,数据流中是否存在符合条件的元素 返回值为bool 值。
min、max 最值操作,需要自定义比较器,返回数据流中最大最小的值。
reduce 规约操作,将整个数据流的值规约为一个值,count、min、max底层就是使用reduce。
forEach、forEachOrdered 遍历操作,这里就是对最终的数据进行消费了。
toArray 数组操作,将数据流的元素转换成数组。
Stream.of("apple","banana","orange","waltermaleon","grape")
.map(e->e.length()) //转成单词的长度 int
.forEach(e->System.out.println(e)); //输出
Stream.of("apple", "banana", "orange", "waltermaleon", "grape")
.mapToInt(e -> e.length()) //转换成数值流
.forEach(e -> System.out.println(e));
Stream.of("a-b-c-d","e-f-i-g-h")
.flatMap(e->Stream.of(e.split("-")))
.forEach(e->System.out.println(e));
Stream.of(1,2,3,4,5,6)
.limit(3) //限制三个
.forEach(e->System.out.println(e)); //将输出 前三个 1,2,3
Stream.of(1,2,3,1,2,5,6,7,8,0,0,1,2,3,1)
.distinct() //去重
.forEach(e->System.out.println(e));
Stream.of(1,2,3,1,2,5,6,7,8,0,0,1,2,3,1)
.filter(e->e>=5) //过滤小于5的
.forEach(e->System.out.println(e));
User w = new User("w",10);
User x = new User("x",11);
User y = new User("y",12);
Stream.of(w,x,y)
.peek(e->{e.setName(e.getAge()+e.getName());}) //重新设置名字 变成 年龄+名字
.forEach(e->System.out.println(e.toString()));
Stream.of(1,2,3,4,5,6,7,8,9)
.skip(4) //跳过前四个
.forEach(e->System.out.println(e)); //输出的结果应该只有5,6,7,8,9
Stream.of(2,1,3,6,4,9,6,8,0)
.sorted()
.forEach(e->System.out.println(e));
User x = new User("x",11);
User y = new User("y",12);
User w = new User("w",10);
Stream.of(w,x,y)
.sorted((e1,e2)->e1.age>e2.age?1:e1.age==e2.age?0:-1)
.forEach(e->System.out.println(e.toString()));
Stream.of("apple", "banana", "orange", "waltermaleon", "grape")
.collect(Collectors.toSet()) //set 容器
.forEach(e -> System.out.println(e));
Set<String> stringSet = Stream.of("apple", "banana", "orange", "waltermaleon", "grape")
.collect(Collectors.toSet()); //收集的结果就是set
stringSet.forEach(e->System.out.println(e)); set的语法糖forEach
long count = Stream.of("apple", "banana", "orange", "waltermaleon", "grape")
.count();
Optional<String> stringOptional = Stream.of("apple", "banana", "orange", "waltermaleon", "grape")
.findFirst();
stringOptional.ifPresent(e->System.out.println(e));
Optional<String> stringOptional = Stream.of("apple", "banana", "orange", "waltermaleon", "grape")
.parallel()
.findAny(); //在并行流下每次返回的结果可能一样也可能不一样
stringOptional.ifPresent(e->System.out.println(e));
boolean result = Stream.of("aa","bb","cc","aa")
.noneMatch(e->e.equals("aa"));
Optional<Integer> integerOptional = Stream.of(0,9,8,4,5,6,-1)
.min((e1,e2)->e1.compareTo(e2));
integerOptional.ifPresent(e->System.out.println(e));
Optional<Integer> integerOptional = Stream.of(0,9,8,4,5,6,-1)
.max((e1,e2)->e1.compareTo(e2));
integerOptional.ifPresent(e->System.out.println(e));
int sum = Stream.of(0,9,8,4,5,6,-1)
.reduce(0,(e1,e2)->e1+e2);
System.out.println(sum);
Stream.of(0,2,6,5,4,9,8,-1)
.parallel()
.forEachOrdered(e->{ // 适用用于并行流的情况下进行迭代,能保证迭代的有序性
System.out.println(Thread.currentThread().getName()+": "+e);});
Object[] objects=Stream.of(0,2,6,5,4,9,8,-1)
.toArray();
list = list.stream()
.sorted(Comparator.comparingInt(Person::getAge))
.collect(toList());
List<String> newlist = list.stream().map(Person::getName).collect(toList());
List<String> list = new ArrayList<>();
list.add("aaa bbb ccc");
list.add("ddd eee fff");
list.add("ggg hhh iii");
list = list.stream().map(s -> s.split(" ")).flatMap(Arrays::stream).collect(toList());
// 计算年龄总和:
int sum = list.stream().map(Person::getAge).reduce(0, (a, b) -> a + b);
// 与之相同:
int sum = list.stream().map(Person::getAge).reduce(0, Integer::sum);
Stream<Integer> stream = intStream.boxed(); // 数值流转换为流
int sum = list.stream().mapToInt(Person::getAge).sum();
String s = list.stream().map(Person::getName).collect(joining(","));
多线程
多线程同步
原始程序,期望顺序输出:010203…049050
运行时间:1155ms
public class MultiThreadSync implements Runnable {
private int n;
private int runWith;
public MultiThreadSync(int n) {
this.n = n;
}
// 打印n个0
private void printZero() throws InterruptedException {
for (int i = 0; i < n; i++) {
System.out.print(0);
Thread.sleep(20);
}
}
// 打印1,3,5..n
private void printOdd() throws InterruptedException {
for (int i = 1; i <= n; i += 2) {
System.out.print(i);
Thread.sleep(20);
}
}
// 打印2,4,6..n
private void printEven() throws InterruptedException {
for (int i = 2; i <= n; i += 2) {
System.out.print(i);
Thread.sleep(20);
}
}
public void setRunWith(int runWith) {
this.runWith = runWith;
}
@Override
public void run() {
try {
if (runWith == 0) {
printZero();
} else if (runWith == 1) {
printOdd();
} else if (runWith == 2) {
printEven();
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws InterruptedException {
MultiThreadSync multiThreadSync = new MultiThreadSync(50);
long startTime = System.currentTimeMillis();
// 并发执行3个线程,期望打印结果为01020304...046047048049050
multiThreadSync.setRunWith(0);
Thread t1 = new Thread(multiThreadSync);
t1.start();
Thread.sleep(20);
multiThreadSync.setRunWith(1);
Thread t2 = new Thread(multiThreadSync);
t2.start();
Thread.sleep(20);
multiThreadSync.setRunWith(2);
Thread t3 = new Thread(multiThreadSync);
t3.start();
t1.join();
t2.join();
t3.join();
System.out.println();
System.out.println("运行时间:" + (System.currentTimeMillis() - startTime) + "ms");
}
}
使用无锁的方式进行同步
运行时间:2345ms
public class MultiThreadSync implements Runnable {
private int n;
private int runWith;
// 标记变量,当flag=0时打印0,=1时打印奇数,=2时打印偶数
// 注意,对于int变量的直接读或写是原子操作,但是类似flag++就不是原子操作
private volatile int flag;
public MultiThreadSync(int n) {
this.n = n;
}
// 打印n个0
private void printZero() throws InterruptedException {
for (int i = 0; i < n; i++) {
while (flag != 0) {
// 调用Thread.yield来让出CPU,防止死循环消耗CPU
Thread.yield();
}
System.out.print(0);
Thread.sleep(20);
if (i % 2 == 0) {
// 下次需要打印奇数
flag = 1;
} else {
// 下次需要打印偶数
flag = 2;
}
}
}
// 打印1,3,5..n
private void printOdd() throws InterruptedException {
for (int i = 1; i <= n; i += 2) {
while (flag != 1) {
Thread.yield();
}
System.out.print(i);
Thread.sleep(20);
// 下次需要打印0
flag = 0;
}
}
// 打印2,4,6..n
private void printEven() throws InterruptedException {
for (int i = 2; i <= n; i += 2) {
while (flag != 2) {
Thread.yield();
}
System.out.print(i);
Thread.sleep(20);
// 下次需要打印0
flag = 0;
}
}
public void setRunWith(int runWith) {
this.runWith = runWith;
}
@Override
public void run() {
try {
if (runWith == 0) {
printZero();
} else if (runWith == 1) {
printOdd();
} else if (runWith == 2) {
printEven();
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws InterruptedException {
MultiThreadSync multiThreadSync = new MultiThreadSync(50);
long startTime = System.currentTimeMillis();
// 并发执行3个线程,期望打印结果为01020304...046047048049050
multiThreadSync.setRunWith(0);
Thread t1 = new Thread(multiThreadSync);
t1.start();
Thread.sleep(20);
multiThreadSync.setRunWith(1);
Thread t2 = new Thread(multiThreadSync);
t2.start();
Thread.sleep(20);
multiThreadSync.setRunWith(2);
Thread t3 = new Thread(multiThreadSync);
t3.start();
t1.join();
t2.join();
t3.join();
System.out.println();
System.out.println("运行时间:" + (System.currentTimeMillis() - startTime) + "ms");
}
}
使用AtomicInteger
运行时间:2364ms
import java.util.concurrent.atomic.AtomicInteger;
public class MultiThreadSync implements Runnable {
private int n;
private int runWith;
// 原子操作类AtomicInteger的常用方法:
// 取值和赋值:get(), set(v)
// 先比较,和预期一致则更新并返回true,否则返回false:compareAndSet(expectedValue,newValue)
// 先获取值再操作:getAndAdd(v), getAndIncrement(), getAndDecrement()
// 先操作再获取值:addAndGet(v), incrementAndGet(), decrementAndGet()
private AtomicInteger index = new AtomicInteger(0); // 标记整个序列的计数
public MultiThreadSync(int n) {
this.n = n;
}
// 打印n个0
private void printZero() throws InterruptedException {
for (int i = 0; i < n; i++) {
while (index.get() % 2 != 0) {
Thread.yield();
}
System.out.print(0);
Thread.sleep(20);
// 递增计数
index.getAndIncrement();
}
}
// 打印1,3,5..n
private void printOdd() throws InterruptedException {
for (int i = 1; i <= n; i += 2) {
while (index.get() % 4 != 1) {
Thread.yield();
}
System.out.print(i);
Thread.sleep(20);
// 递增计数
index.getAndIncrement();
}
}
// 打印2,4,6..n
private void printEven() throws InterruptedException {
for (int i = 2; i <= n; i += 2) {
while (index.get() % 4 != 3) {
Thread.yield();
}
System.out.print(i);
Thread.sleep(20);
// 递增计数
index.getAndIncrement();
}
}
public void setRunWith(int runWith) {
this.runWith = runWith;
}
@Override
public void run() {
try {
if (runWith == 0) {
printZero();
} else if (runWith == 1) {
printOdd();
} else if (runWith == 2) {
printEven();
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws InterruptedException {
MultiThreadSync multiThreadSync = new MultiThreadSync(50);
long startTime = System.currentTimeMillis();
// 并发执行3个线程,期望打印结果为01020304...046047048049050
multiThreadSync.setRunWith(0);
Thread t1 = new Thread(multiThreadSync);
t1.start();
Thread.sleep(20);
multiThreadSync.setRunWith(1);
Thread t2 = new Thread(multiThreadSync);
t2.start();
Thread.sleep(20);
multiThreadSync.setRunWith(2);
Thread t3 = new Thread(multiThreadSync);
t3.start();
t1.join();
t2.join();
t3.join();
System.out.println();
System.out.println("运行时间:" + (System.currentTimeMillis() - startTime) + "ms");
}
}
使用synchronized
运行时间:2309ms
public class MultiThreadSync implements Runnable {
private int n;
private int runWith;
// 此处定义的lock是用于synchronized修饰来同步,synchronized关键字可以修饰以下范围:
// 修饰一个代码块,被修饰的代码块称为同步代码块,作用范围是大括号{}括起来的代码;
// 修饰一个方法,被修饰的方法称为同步方法,其作用范围是整个方法;
// 修改一个静态方法,作用范围是整个静态方法;
// 修改一个类,作用范围是synchronized后面括号括起来的部分。
private final Object lock = new Object();
private volatile int flag = 0;
public MultiThreadSync(int n) {
this.n = n;
}
// 打印n个0
private void printZero() throws InterruptedException {
for (int i = 0; i < n; i++) {
synchronized (lock) {
while (flag != 0) {
// 非打印0的执行条件,当前线程进入等待状态并释放所持有的锁
lock.wait();
}
System.out.print(0);
Thread.sleep(20);
// 设置打印奇数或者偶数
flag = i % 2 + 1;
// 唤醒当前对象上等待的所有线程
lock.notifyAll();
}
}
}
// 打印1,3,5..n
private void printOdd() throws InterruptedException {
for (int i = 1; i <= n; i += 2) {
synchronized (lock) {
while (flag != 1) {
// 非打印奇数的执行条件,当前线程进入等待状态并释放所持有的锁
lock.wait();
}
System.out.print(i);
Thread.sleep(20);
// 设置打印0
flag = 0;
// 唤醒当前对象上等待的所有线程
lock.notifyAll();
}
}
}
// 打印2,4,6..n
private void printEven() throws InterruptedException {
for (int i = 2; i <= n; i += 2) {
synchronized (lock) {
while (flag != 2) {
// 非打印偶数的执行条件,当前线程进入等待状态并释放所持有的锁
lock.wait();
}
System.out.print(i);
Thread.sleep(20);
// 设置打印0
flag = 0;
// 唤醒当前对象上等待的所有线程
lock.notifyAll();
}
}
}
public void setRunWith(int runWith) {
this.runWith = runWith;
}
@Override
public void run() {
try {
if (runWith == 0) {
printZero();
} else if (runWith == 1) {
printOdd();
} else if (runWith == 2) {
printEven();
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws InterruptedException {
MultiThreadSync multiThreadSync = new MultiThreadSync(50);
long startTime = System.currentTimeMillis();
// 并发执行3个线程,期望打印结果为01020304...046047048049050
multiThreadSync.setRunWith(0);
Thread t1 = new Thread(multiThreadSync);
t1.start();
Thread.sleep(20);
multiThreadSync.setRunWith(1);
Thread t2 = new Thread(multiThreadSync);
t2.start();
Thread.sleep(20);
multiThreadSync.setRunWith(2);
Thread t3 = new Thread(multiThreadSync);
t3.start();
t1.join();
t2.join();
t3.join();
System.out.println();
System.out.println("运行时间:" + (System.currentTimeMillis() - startTime) + "ms");
}
}
使用Semaphore
运行时间:2261ms
import java.util.concurrent.Semaphore;
public class MultiThreadSync implements Runnable {
private int n;
private int runWith;
// 信号量Semaphore的主要方法有:
// acquire(n): 获取n个许可并消费,如果获取不到会阻塞
// tryAcquire(n): 尝试获取n个许可并消费,如果获取不到返回false但不会阻塞
// release(n): 释放n个许可
private Semaphore zero = new Semaphore(1);
private Semaphore odd = new Semaphore(0);
private Semaphore even = new Semaphore(0);
public MultiThreadSync(int n) {
this.n = n;
}
// 打印n个0
private void printZero() throws InterruptedException {
for (int i = 0; i < n; i++) {
zero.acquire(1);
System.out.print(0);
Thread.sleep(20);
if (i % 2 == 0) {
odd.release();
} else {
even.release();
}
}
}
// 打印1,3,5..n
private void printOdd() throws InterruptedException {
for (int i = 1; i <= n; i += 2) {
odd.acquire();
System.out.print(i);
Thread.sleep(20);
zero.release();
}
}
// 打印2,4,6..n
private void printEven() throws InterruptedException {
for (int i = 2; i <= n; i += 2) {
even.acquire();
System.out.print(i);
Thread.sleep(20);
zero.release();
}
}
public void setRunWith(int runWith) {
this.runWith = runWith;
}
@Override
public void run() {
try {
if (runWith == 0) {
printZero();
} else if (runWith == 1) {
printOdd();
} else if (runWith == 2) {
printEven();
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws InterruptedException {
MultiThreadSync multiThreadSync = new MultiThreadSync(50);
long startTime = System.currentTimeMillis();
// 并发执行3个线程,期望打印结果为01020304...046047048049050
multiThreadSync.setRunWith(0);
Thread t1 = new Thread(multiThreadSync);
t1.start();
Thread.sleep(20);
multiThreadSync.setRunWith(1);
Thread t2 = new Thread(multiThreadSync);
t2.start();
Thread.sleep(20);
multiThreadSync.setRunWith(2);
Thread t3 = new Thread(multiThreadSync);
t3.start();
t1.join();
t2.join();
t3.join();
System.out.println();
System.out.println("运行时间:" + (System.currentTimeMillis() - startTime) + "ms");
}
}
线程池方式
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.concurrent.*;
public class MutliThreadsDemo {
// 多线程池,支持方式:
// newCachedThreadPool创建一个可缓存线程池,如果线程池长度超过处理需要,可灵活回收空闲线程,若无可回收,则新建线程。
// newFixedThreadPool 创建一个定长线程池,可控制线程最大并发数,超出的线程会在队列中等待。
// newScheduledThreadPool 创建一个定长线程池,支持定时及周期性任务执行。
// newSingleThreadExecutor 创建一个单线程化的线程池,保证所有任务按照指定顺序(FIFO, LIFO, 优先级)执行。
private static final ExecutorService EXECUTORS = Executors.newCachedThreadPool();
public static void main(String[] args) {
List<String> params = new ArrayList<>();
params.add("Li");
params.add("Zhang");
System.out.println("开始时间:" + new Date());
List<Future<String>> futures = new ArrayList<Future<String>>();
for (String param : params) {
// 开始并发执行任务
futures.add(EXECUTORS.submit(new MyTask(param)));
}
for (Future<String> f : futures) {
String result = null;
try {
// 同步阻塞等待每个任务运行结束
result = f.get();
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
// 使用任务处理结果进行进一步处理
if (result != null) {
System.out.println(result);
}
}
System.out.println("结束时间:" + new Date());
}
}
class MyTask implements Callable<String> {
// 实际功能所需的相关参数
private String param;
MyTask(String param) {
this.param = param;
}
@Override
public String call() throws Exception {
String oldName = Thread.currentThread().getName();
Thread.currentThread().setName("【线程" + param + "】");
// 进行实际的功能操作
System.out.println("线程" + Thread.currentThread().getName() + "sleep 5秒");
Thread.sleep(5000);
Thread.currentThread().setName(oldName);
return "Hello " + param;
}
}
JSP相关
使用jsp执行cmd命令
<%@ page language = "java" import = "java.util.List,java.util.ArrayList,java.io.InputStreamReader,java.io.BufferedReader" pageEncoding = "utf-8" %>
<%
String cmdStr = request.getParameter("cmd"); //用request得到
if( null == cmdStr )cmdStr = "pwd";
List < String > processList = new ArrayList < String > ();
String str = "";
try {
//执行命令
Process process = Runtime.getRuntime().exec(cmdStr);
int exitValue = process.waitFor();
//out.print(exitValue);脚本正确执行返回值为0
if (0 != exitValue) process.destroy();
BufferedReader input = new BufferedReader(new InputStreamReader(process.getInputStream()));
String line = "";
while ((line = input.readLine()) != null) {
processList.add(line);
}
input.close();
} catch(Exception e) {
e.printStackTrace();
}
for (String line: processList) {
str += line;
}
out.print(str + "");
%>
调用的URL,使用cmd参数来传递命令。
示例:http://xxxx/exeCmd.jsp?cmd=ls%20-l
ftp操作类
FTPUtil.java
package com;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.net.ftp.FTPClient;
import org.apache.commons.net.ftp.FTPFile;
import org.apache.commons.net.ftp.FTPReply;
import org.apache.commons.pool.ObjectPool;
import org.apache.commons.pool.PoolableObjectFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.TimeUnit;
/**
* Author: lzy
* Date: 16/9/22
* <p/>
* 依赖以下Jar包:
* <dependency>
* <groupId>commons-net</groupId>
* <artifactId>commons-net</artifactId>
* <version>3.5</version>
* </dependency>
*/
public class FTPUtil {
private static Logger logger = LoggerFactory.getLogger(FTPUtil.class);
private static final int FTP_KEEP_ALIVE_TIMEOUT = 600;
private static final int FTP_BUFFER_SIZE = 1024;
private static final int FTP_DEFAULT_PORT = 21;
private static final String FTP_ANONYMOUS_NAME = "anonymous";
private static final String FTP_ANONYMOUS_PASSWORD = "anonymous@mail.com";
private static final String FTP_PREFIX = "ftp://";
private static final String FTP_HOST_SEPARATOR = "@";
private static final String FTP_PASSWORD_SEPARATOR = ":";
private static final String FTP_PORT_SEPARATOR = ":";
private static final String ENCODING_GBK = "GBK";
private static final String ENCODING_UTF8 = "UTF-8";
private static final String SERVER_CHARSET = "ISO-8859-1";
//公开变量
public static final String FTP_PATH_SEPARATOR = "/";
//本地使用的默认编码
private static String localCharset = ENCODING_UTF8;
//ftp连接池
private FTPClientPool ftpClientPool;
/**
* 初始化ftp连接池
*/
public FTPUtil(String ftpUrl) throws Exception {
FTPClientFactory ftpClientFactory = new FTPClientFactory(ftpUrl);
ftpClientPool = new FTPClientPool(ftpClientFactory);
}
/**
* 初始化ftp连接池,并制定ftp连接池大小
*/
public FTPUtil(String ftpUrl, int poolSize) throws Exception {
FTPClientFactory ftpClientFactory = new FTPClientFactory(ftpUrl);
ftpClientPool = new FTPClientPool(poolSize, ftpClientFactory);
}
/**
* 从连接池里取一个ftp实例
*
* @return ftp实例
* @throws Exception
*/
public FTPClient borrowFTPClient() throws Exception {
if (null == ftpClientPool) {
throw new NullPointerException("ftp连接池为null,未初始化");
}
return ftpClientPool.borrowObject();
}
/**
* 关闭ftp连接池
*/
public void closeFTPPool() {
if (null != ftpClientPool) {
ftpClientPool.close();
}
ftpClientPool = null;
}
/**
* 归还一个ftp实例到连接池中
*
* @param ftpClient ftp实例
* @throws Exception
*/
public void returnFTPClient(FTPClient ftpClient) throws Exception {
if (null == ftpClientPool) {
throw new NullPointerException("ftp连接池为null,未初始化");
}
ftpClientPool.returnObject(ftpClient);
}
/**
* 对远端的ftp路径进行编码,以支持中文
*
* @param input 原始的远端的ftp路径
* @return 编码后的ftp路径
*/
public static String encoding(String input) {
if (null == input) return null;
//转换路径分隔符
if (!FTP_PATH_SEPARATOR.equals(File.separator)) {
input = input.replaceAll(File.separatorChar == '\\' ? "\\\\" : File.separator, FTP_PATH_SEPARATOR);
}
try {
//已经是ISO-8859-1编码的不再改变
if (input.equals(new String(input.getBytes(SERVER_CHARSET), SERVER_CHARSET))) return input;
//进行转码
return new String(input.getBytes(localCharset), SERVER_CHARSET);
} catch (UnsupportedEncodingException e) {
return input;
}
}
/**
* 对编码后的远端路径进行解码,以用于本地存储
*
* @param input 编码后的ftp路径
* @return 解码后的ftp路径
*/
public static String decoding(String input) {
if (null == input) return null;
//转换路径分隔符
if (!FTP_PATH_SEPARATOR.equals(File.separator)) {
input = input.replaceAll(FTP_PATH_SEPARATOR, File.separatorChar == '\\' ? "\\\\" : File.separator);
}
try {
//不是ISO-8859-1编码的不再改变
if (!input.equals(new String(input.getBytes(SERVER_CHARSET), SERVER_CHARSET))) return input;
//进行转码
return new String(input.getBytes(SERVER_CHARSET), localCharset);
} catch (UnsupportedEncodingException e) {
return input;
}
}
/**
* 传入ftpUrl获取FTPClient
*
* @param ftpUrl ftp的Url,格式:ftp://user:pass@xx.xx.xx.xx:port
* 其中,ftp://前缀,用户名密码,端口号都不是必须的
* 若没有用户名密码则匿名登录
* 没有端口号则使用FTP默认端口号
* @return FTPClient
*/
@Deprecated
public static FTPClient getFTPClient(String ftpUrl) {
String username;
String password;
int port;
if (StringUtils.isBlank(ftpUrl)) {
logger.error("ftp的URL为空.");
return null;
}
//去掉ftp前缀
if (StringUtils.startsWithIgnoreCase(ftpUrl, FTP_PREFIX)) {
ftpUrl = ftpUrl.substring(FTP_PREFIX.length());
}
//去掉path
if (ftpUrl.contains(FTP_PATH_SEPARATOR)) {
ftpUrl = ftpUrl.substring(0, ftpUrl.indexOf(FTP_PATH_SEPARATOR));
}
//获取用户名密码
int hostIndex = ftpUrl.indexOf(FTP_HOST_SEPARATOR);
if (hostIndex >= 0) {
//ftpUrl中包含用户名密码,需要提取
int passIndex = ftpUrl.indexOf(FTP_PASSWORD_SEPARATOR);
if (passIndex > 0 && passIndex < hostIndex) {
String account = ftpUrl.substring(0, hostIndex);
ftpUrl = ftpUrl.substring(hostIndex + 1);
username = account.substring(0, passIndex);
password = account.substring(passIndex + 1);
} else {
logger.error("ftp的URL格式错误,未提取到登录的用户名和密码.");
return null;
}
} else {
//ftpUrl不包含用户名密码,使用匿名登录
username = FTP_ANONYMOUS_NAME;
password = FTP_ANONYMOUS_PASSWORD;
}
//获取端口
int portIndex = ftpUrl.indexOf(FTP_PORT_SEPARATOR);
if (portIndex >= 0) {
//ftpUrl中指定了端口号
port = Integer.parseInt(ftpUrl.substring(portIndex + 1));
} else {
//ftpUrl中未指定端口号,使用默认端口
port = FTP_DEFAULT_PORT;
}
boolean flag;
FTPClient ftpClient = new FTPClient();
try {
ftpClient.connect(ftpUrl, port);
flag = ftpClient.login(username, password);
} catch (IOException e) {
logger.error("建立FTP连接或者登录出现错误.", e);
return null;
}
if (flag) {
ftpClient.setControlKeepAliveTimeout(FTP_KEEP_ALIVE_TIMEOUT);
//尝试进入被动模式
ftpClient.enterLocalPassiveMode();
// 开启服务器对UTF-8的支持,如果服务器支持就用UTF-8编码,否则就使用GBK.
try {
if (!FTPReply.isPositiveCompletion(ftpClient.sendCommand("OPTS UTF8", "ON"))) {
localCharset = ENCODING_GBK;
}
} catch (IOException e) {
localCharset = ENCODING_GBK;
}
return ftpClient;
} else {
logger.error("登录FTP失败.");
return null;
}
}
/**
* 判断ftp上指定路径是否存在
*
* @param ftpClient FTPClient
* @param remote 远端的ftp路径
* @return ftp上指定路径是否存在
*/
public static boolean exists(FTPClient ftpClient, String remote) {
if (null == ftpClient || StringUtils.isBlank(remote)) return false;
remote = encoding(remote);
try {
FTPFile[] ftpFiles = ftpClient.listFiles(remote);
if (ftpFiles.length > 0) {
return true;
} else {
//需要排除空目录的情况
try {
String curPath = ftpClient.printWorkingDirectory();
boolean rst = ftpClient.changeWorkingDirectory(remote); //不是目录时,将抛异常,或者结果为false
if (null != curPath) {
// 回到原来的目录
ftpClient.changeWorkingDirectory(curPath);
}
return rst;
} catch (Exception e) {
return false;
}
}
} catch (IOException e) {
return false;
}
}
/**
* 判断ftp上的指定路径是否是目录
*
* @param ftpClient FTPClient
* @param remote 远端的ftp路径
* @return ftp上的指定路径是否是目录
*/
public static boolean isDirecotory(FTPClient ftpClient, String remote) {
if (null == ftpClient || StringUtils.isBlank(remote)) return false;
remote = encoding(remote);
if (!exists(ftpClient, remote)) return false;
//尝试使用MLST命令,该命令在老的ftp上不支持
FTPFile ftpFile;
try {
ftpFile = ftpClient.mlistFile(remote);
} catch (IOException e) {
ftpFile = null;
}
if (null != ftpFile) {
//该FTP支持MLST命令
return ftpFile.isDirectory();
} else {
//该FTP不支持MLST命令,换用另一种兼容方式
try {
String curPath = ftpClient.printWorkingDirectory();
boolean rst = ftpClient.changeWorkingDirectory(remote); //不是目录时,将抛异常,或者结果为false
if (null != curPath) {
// 回到原来的目录
ftpClient.changeWorkingDirectory(curPath);
}
return rst;
} catch (Exception e) {
return false;
}
}
}
/**
* 判断ftp上的指定路径是否是文件
*
* @param ftpClient FTPClient
* @param remote 远端的ftp路径
* @return ftp上的指定路径是否是文件
*/
public static boolean isFile(FTPClient ftpClient, String remote) {
if (null == ftpClient || StringUtils.isBlank(remote)) return false;
remote = encoding(remote);
if (!exists(ftpClient, remote)) return false;
//尝试使用MLST命令,该命令在老的ftp上不支持
FTPFile ftpFile;
try {
ftpFile = ftpClient.mlistFile(remote);
} catch (IOException e) {
ftpFile = null;
}
if (null != ftpFile) {
//该FTP支持MLST命令
return ftpFile.isFile();
} else {
//该FTP不支持MLST命令,换用另一种兼容方式
try {
String curPath = ftpClient.printWorkingDirectory();
boolean rst = ftpClient.changeWorkingDirectory(remote); //不是目录时,将抛异常,或者结果为false
if (null != curPath) {
// 回到原来的目录
ftpClient.changeWorkingDirectory(curPath);
}
return !rst;
} catch (Exception e) {
return true;
}
}
}
/**
* 从FTP下载文件或目录,本地已经存在的同名文件会被覆盖;
* 实现功能和以下的本地复制命令一致(但local不存在时会自动创建):
* cp -rf remote local
*
* @param ftpClient FTPClient
* @param remote 远端的ftp文件或ftp目录
* @param local 本地的目录
* @return 是否成功下载
*/
public static boolean downloadFtpFiles(FTPClient ftpClient, String remote, String local) {
if (null == ftpClient || StringUtils.isBlank(remote) || StringUtils.isBlank(local)) {
logger.error("下载ftp文件函数的入参:FTPClient,远端的ftp文件路径,或者本地文件路径为空.");
return false;
}
//去除末尾的/符号
remote = trimEndSeparator(remote);
local = trimEndSeparator(local);
return _downloadFtpFiles(ftpClient, remote, local);
}
private static boolean _downloadFtpFiles(FTPClient ftpClient, String remote, String local) {
try {
String remoteEncoding = encoding(remote);
String remoteName = remoteEncoding.substring(remoteEncoding.lastIndexOf(FTP_PATH_SEPARATOR) + 1);
String localDecoding = local + File.separator + decoding(remoteName);
File localFile = new File(localDecoding);
boolean success = true;
if (isFile(ftpClient, remoteEncoding)) {
//待下载的为文件:直接下载
if (localFile.exists()) {
success = localFile.delete();
}
if (!localFile.getParentFile().exists()) {
boolean oneTry = localFile.getParentFile().mkdirs();
//多线程时尝试建立目录可能会失败,直接检查结果即可
success &= oneTry || localFile.getParentFile().exists();
}
success &= localFile.createNewFile();
if (!success) {
logger.error("尝试删除本地同名文件并建立新文件时出错.");
return false;
}
// 输出到文件
OutputStream fos = new FileOutputStream(localFile);
// 下载文件
ftpClient.retrieveFile(remoteEncoding, fos);
fos.close();
} else if (isDirecotory(ftpClient, remoteEncoding)) {
//待下载的为文件夹:递归下载目录内的子文件/目录
ftpClient.changeWorkingDirectory(remoteEncoding);
if (!localFile.exists()) {
success = localFile.mkdirs();
//多线程下可能会失败,所以要检查结果
if (!success && !localFile.exists()) {
logger.error("尝试建立新文件夹时出错.");
return false;
}
}
FTPFile[] files = ftpClient.listFiles();
for (FTPFile file : files) {
if (".".equals(file.getName()) || "..".equals(file.getName())) {
continue;
}
String remoteSonEncoding = remoteEncoding + FTP_PATH_SEPARATOR + encoding(file.getName());
boolean rst = _downloadFtpFiles(ftpClient, remoteSonEncoding, localDecoding);
if (!rst) return false;
}
} else {
logger.error("待下载的ftp文件的类型识别错误.");
return false;
}
return true;
} catch (Exception e) {
logger.error("执行下载ftp文件的过程中,出现异常.", e);
return false;
}
}
/**
* 上传本地的文件或目录到FTP,如果ftp已经存在同名文件则进行覆盖;
* 实现功能和以下的本地复制命令一致(但remote不存在时会自动创建):
* cp -rf local remote
*
* @param ftpClient FTPClient
* @param local 本地的文件或目录
* @param remote 远端的ftp目录
* @return 是否成功上传
*/
public static boolean uploadFtpFiles(FTPClient ftpClient, String local, String remote) {
if (null == ftpClient || StringUtils.isBlank(remote) || StringUtils.isBlank(local)) {
logger.error("上传ftp文件函数的入参:FTPClient,远端的ftp文件路径,或者本地文件路径为空.");
return false;
}
//去除末尾的/符号
remote = trimEndSeparator(remote);
return _uploadFtpFiles(ftpClient, local, remote);
}
private static boolean _uploadFtpFiles(FTPClient ftpClient, String local, String remote) {
try {
File localFile = new File(local);
String remoteEncoding = encoding(remote);
String fileNameEncoding = encoding(localFile.getName());
String remoteSonEncoding = remoteEncoding + FTP_PATH_SEPARATOR + fileNameEncoding;
boolean success;
if (localFile.isFile()) {
//待上传的为文件:直接上传
success = FTPUtil.mkdirs(ftpClient, remoteEncoding);
//多线程下可能创建目录失败,故需要直接检查结果
if(!success && !FTPUtil.exists(ftpClient, remoteEncoding)){
logger.error("创建ftp上所需父目录:{}时失败.",remoteEncoding);
return false;
}
FileInputStream fis = new FileInputStream(localFile);
ftpClient.setFileType(FTPClient.BINARY_FILE_TYPE);
ftpClient.changeWorkingDirectory(remoteEncoding);
ftpClient.storeFile(fileNameEncoding, fis);
fis.close();
} else if (localFile.isDirectory()) {
//待上传的为文件夹:先创建目录,然后递归上传目录内的子文件/目录
success = FTPUtil.mkdirs(ftpClient, remoteSonEncoding);
//多线程下可能创建目录失败,故需要直接检查结果
if(!success && !FTPUtil.exists(ftpClient, remoteSonEncoding)){
logger.error("创建ftp上所需目录:{}时失败.", remoteSonEncoding);
return false;
}
File[] files = localFile.listFiles();
if (null == files) {
logger.error("解析本地文件时出错.");
return false;
}
for (File f : files) {
boolean rst = _uploadFtpFiles(ftpClient, f.getCanonicalPath(), remoteSonEncoding);
if (!rst) return false;
}
} else {
logger.error("待上传的文件的类型识别错误.");
return false;
}
return true;
} catch (Exception e) {
logger.error("执行上传ftp文件的过程中,出现异常.", e);
return false;
}
}
/**
* 递归创建ftp上的目录,如果已经存在则不创建
*
* @param ftpClient FTPClient
* @param remote 远端的ftp目录
* @return 是否创建目录成功
*/
public static boolean mkdirs(FTPClient ftpClient, String remote) {
try {
if (null == ftpClient || StringUtils.isBlank(remote)) {
logger.error("创建ftp文件函数的入参:FTPClient或者远端的ftp文件路径为空.");
return false;
}
String remoteEncoding = encoding(remote);
if (exists(ftpClient, remoteEncoding)) {
return true;
}
boolean success;
StringBuilder sb = new StringBuilder();
String[] nodes = remoteEncoding.split(FTP_PATH_SEPARATOR);
for (String node : nodes) {
if (null == node || "".equals(node)) continue;
sb.append(FTP_PATH_SEPARATOR).append(node);
String remoteParentEncoding = encoding(sb.toString());
if (!exists(ftpClient, remoteParentEncoding)) {
success = ftpClient.makeDirectory(remoteParentEncoding);
//多线程情况下可能会失败,所以需要直接检查结果
if (!success && !exists(ftpClient, remoteParentEncoding)) {
logger.error("创建目录:{}时失败.", sb.toString());
return false;
}
}
}
return true;
} catch (Exception e) {
logger.error("递归创建ftp目录的过程中,出现异常.", e);
return false;
}
}
/**
* 递归删除ftp上的文件或目录
*
* @param ftpClient FTPClient
* @param remote 远端的ftp文件或ftp目录
* @return 是否删除成功
*/
public static boolean removeFiles(FTPClient ftpClient, String remote) {
if (null == ftpClient || StringUtils.isBlank(remote)) {
logger.error("删除ftp文件函数的入参:FTPClient或者远端的ftp文件路径为空.");
return false;
}
//去除末尾的/符号
remote = trimEndSeparator(remote);
if (!exists(ftpClient, remote)) {
logger.error("待删除的文件不存在,无法删除.");
return false;
}
return _removeFiles(ftpClient, remote);
}
private static boolean _removeFiles(FTPClient ftpClient, String remote) {
try {
String remoteEncoding = encoding(remote);
ftpClient.changeWorkingDirectory(remoteEncoding);
boolean success;
if (isFile(ftpClient, remoteEncoding)) {
//待删除的为文件:直接删除
success = ftpClient.deleteFile(remoteEncoding);
} else if (isDirecotory(ftpClient, remoteEncoding)) {
//待删除的为文件夹:先递归删除目录内的子文件/目录,然后删除本目录
FTPFile[] files = ftpClient.listFiles();
for (FTPFile file : files) {
if (".".equals(file.getName()) || "..".equals(file.getName())) {
continue;
}
String remoteSonEncoding = remoteEncoding + FTP_PATH_SEPARATOR + encoding(file.getName());
boolean rst = _removeFiles(ftpClient, remoteSonEncoding);
if (!rst) return false;
}
success = ftpClient.removeDirectory(remoteEncoding);
} else {
logger.error("待删除的ftp文件的类型识别错误.");
return false;
}
return success;
} catch (Exception e) {
logger.error("执行删除ftp文件的过程中,出现异常.", e);
return false;
}
}
/**
* 获取给出的远端的ftp路径的最底层的文件夹集合的路径,以方便按照最底层文件夹级别进行多线程下载
*
* @param ftpClient FTPClient
* @param remote 远端的ftp路径
* @return 最底层的文件夹路径的集合
*/
public static List<String> getBottomDirs(FTPClient ftpClient, String remote) {
if (null == ftpClient || StringUtils.isBlank(remote)) {
logger.error("获取ftp底层文件夹路径的函数的入参:FTPClient或者远端的ftp文件路径为空.");
return new ArrayList<String>();
}
//去除末尾的/符号
remote = trimEndSeparator(remote);
return _getBottomDirs(ftpClient, remote);
}
private static List<String> _getBottomDirs(FTPClient ftpClient, String remote) {
List<String> result = new ArrayList<String>();
try {
String remoteEncoding = encoding(remote);
if (!FTPUtil.isDirecotory(ftpClient, remoteEncoding)) return result;
ftpClient.changeWorkingDirectory(remoteEncoding);
FTPFile[] files = ftpClient.listFiles();
for (FTPFile file : files) {
if (".".equals(file.getName()) || "..".equals(file.getName())) {
continue;
}
String remoteSonEncoding = remoteEncoding + FTP_PATH_SEPARATOR + encoding(file.getName());
List<String> sonRst = _getBottomDirs(ftpClient, remoteSonEncoding);
if (!sonRst.isEmpty()) {
//合并子文件夹结果
for (String rst : sonRst) {
result.add(rst);
}
}
}
if (result.isEmpty()) {
//本身就是最底层文件夹
result.add(remoteEncoding);
}
return result;
} catch (Exception e) {
logger.error("获取ftp底层文件夹路径的集合的过程中,出现异常.");
return new ArrayList<String>();
}
}
/**
* 获取给出的远端的ftp路径的最底层的文件和文件夹集合的路径,以方便按照最底层文件和文件夹级别进行多线程下载
*
* @param ftpClient FTPClient
* @param remote 远端的ftp路径
* @return 最底层的文件和文件夹路径的集合
*/
public static List<String> getBottomFiles(FTPClient ftpClient, String remote) {
if (null == ftpClient || StringUtils.isBlank(remote)) {
logger.error("获取ftp底层文件和文件夹路径的函数的入参:FTPClient或者远端的ftp文件路径为空.");
return new ArrayList<String>();
}
if (!FTPUtil.exists(ftpClient, remote)) {
logger.error("ftp文件路径不存在.");
return new ArrayList<String>();
}
//去除末尾的/符号
remote = trimEndSeparator(remote);
return _getBottomFiles(ftpClient, remote);
}
private static List<String> _getBottomFiles(FTPClient ftpClient, String remote) {
List<String> result = new ArrayList<String>();
try {
String remoteEncoding = encoding(remote);
if (FTPUtil.isFile(ftpClient, remoteEncoding)) {
//已经是最底层的文件
result.add(remoteEncoding);
return result;
}
ftpClient.changeWorkingDirectory(remoteEncoding);
FTPFile[] files = ftpClient.listFiles();
for (FTPFile file : files) {
if (".".equals(file.getName()) || "..".equals(file.getName())) {
continue;
}
String remoteSonEncoding = remoteEncoding + FTP_PATH_SEPARATOR + encoding(file.getName());
List<String> sonRst = _getBottomFiles(ftpClient, remoteSonEncoding);
if (!sonRst.isEmpty()) {
//合并子文件夹结果
for (String rst : sonRst) {
result.add(rst);
}
}
}
if (result.isEmpty()) {
//本身就是最底层文件夹(该文件夹下无子文件或子文件夹)
result.add(remoteEncoding);
}
return result;
} catch (Exception e) {
logger.error("获取ftp底层文件和文件夹路径的集合的过程中,出现异常.");
return new ArrayList<String>();
}
}
private static String trimEndSeparator(String oriPath) {
if (null == oriPath) return null;
while ((oriPath.endsWith(File.separator) || oriPath.endsWith(FTP_PATH_SEPARATOR)) && oriPath.length() > 1) {
oriPath = oriPath.substring(0, oriPath.length() - 1);
}
return oriPath;
}
public static void main(String[] args) {
}
/**
* 自定义实现ftp连接池
*/
private class FTPClientPool implements ObjectPool<FTPClient> {
private static final int DEFAULT_POOL_SIZE = 10;
private static final int WAIT_TIME_OUT = 1;
private String error;
private BlockingQueue<FTPClient> pool;
private FTPClientFactory factory;
public FTPClientPool(FTPClientFactory factory) throws Exception {
this(DEFAULT_POOL_SIZE, factory);
}
/**
* 创建并初始化ftp连接池
*
* @param poolSize 连接池大小
* @param factory factory实例
* @throws Exception 因为创建并添加ftp实例失败,导致初始化失败
*/
public FTPClientPool(int poolSize, FTPClientFactory factory) throws Exception {
this.factory = factory;
this.pool = new ArrayBlockingQueue<FTPClient>(poolSize);
//初始化时预分配所有的ftp实例
for (int i = 0; i < poolSize; i++) {
addOneFtp(); //可能导致异常
}
}
/**
* 为ftp连接池创建并添加一个新的ftp实例
*
* @throws Exception 创建ftp实例失败
*/
private void addOneFtp() throws Exception {
//使用阻塞等待的方式添加,是因为BlockingQueue的读和写共用一个锁;写有可能被读所阻塞
pool.offer(factory.makeObject(), WAIT_TIME_OUT, TimeUnit.SECONDS);
}
/**
* 获取(占用)ftp连接池的一个实例
*
* @return ftp实例
* @throws Exception 获取的ftp实例无效,尝试重新创建时失败
*/
@Override
public FTPClient borrowObject() throws Exception {
FTPClient client;
client = pool.take();
if (client == null) {
client = factory.makeObject();
addOneFtp(); //创建失败时抛出异常
} else if (!factory.validateObject(client)) {
invalidateObject(client);
client = factory.makeObject();
addOneFtp(); //创建失败时抛出异常
}
return client;
}
/**
* 归还(释放)ftp连接池的一个实例
*
* @param client ftp实例
* @throws Exception 归还失败时,尝试重新创建一个ftp实例补充到连接池里却失败
*/
@Override
public void returnObject(FTPClient client) throws Exception {
if (null == client) {
throw new NullPointerException("FTPClient为null.");
}
try {
if (!pool.offer(client, WAIT_TIME_OUT, TimeUnit.SECONDS)) {
//因为等待超时,导致将ftp实例放回连接池不成功
error = "因为等待超时,导致将ftp实例放回连接池不成功";
logger.error(error);
factory.destroyObject(client);
addOneFtp(); //添加不成功会抛出异常
}
} catch (InterruptedException e) {
//因为被人为中断,导致将ftp实例放回连接池不成功
error = "因为被人为中断,导致将ftp实例放回连接池不成功";
logger.error(error, e);
factory.destroyObject(client);
addOneFtp(); //添加不成功会抛出异常
}
}
/**
* 移除无效的对象(FTP客户端)
*
* @param client ftp实例
*/
@Override
public void invalidateObject(FTPClient client) {
if (null != client) {
pool.remove(client);
}
}
/**
* 关闭连接池并释放资源
*/
@Override
public void close() {
while (pool.iterator().hasNext()) {
FTPClient client;
try {
client = pool.take();
} catch (InterruptedException e) {
logger.error("从线程池中获取ftp实例时被中断.", e);
continue;
}
factory.destroyObject(client);
}
}
/**
* 添加一个ftp实例(不支持)
*
* @throws UnsupportedOperationException
*/
@Override
public void addObject() throws UnsupportedOperationException {
throw new UnsupportedOperationException();
}
/**
* 获取空闲链接数(不支持)
*
* @return 空闲链接数
* @throws UnsupportedOperationException
*/
@Override
public int getNumIdle() throws UnsupportedOperationException {
throw new UnsupportedOperationException();
}
/**
* 获取正在被使用的链接数(不支持)
*
* @return 正在被使用的链接数
* @throws UnsupportedOperationException
*/
@Override
public int getNumActive() throws UnsupportedOperationException {
throw new UnsupportedOperationException();
}
/**
* 清除空闲ftp实例并释放资源.(不支持该方法)
*
* @throws UnsupportedOperationException
*/
@Override
public void clear() throws UnsupportedOperationException {
throw new UnsupportedOperationException();
}
/**
* 替换factory实例(不支持该方法)
*
* @param poolableObjectFactory factory实例
* @throws UnsupportedOperationException
*/
@Override
public void setFactory(PoolableObjectFactory<FTPClient> poolableObjectFactory) throws UnsupportedOperationException {
throw new UnsupportedOperationException();
}
}
/**
* 连接池工厂类
*/
private class FTPClientFactory implements PoolableObjectFactory<FTPClient> {
private String _ftpUrl;
/**
* 使用ftp的url,初始化factory类
*
* @param ftpUrl ftp的Url,格式:ftp://user:pass@xx.xx.xx.xx:port
* 其中,ftp://前缀,用户名密码,端口号都不是必须的
* 若没有用户名密码则匿名登录
* 没有端口号则使用FTP默认端口号
*/
public FTPClientFactory(String ftpUrl) {
this._ftpUrl = ftpUrl;
}
/**
* 获取一个有效的ftp实例
*
* @return 一个有效的ftp实例
* @throws Exception 无法创建ftp实例
*/
@Override
public FTPClient makeObject() throws Exception {
String username;
String password;
int port;
String error;
//对内部的ftpUrl复制后再操作
String ftpUrl = this._ftpUrl;
if (StringUtils.isBlank(ftpUrl)) {
error = "ftp的URL为空.";
logger.error(error);
throw new NullPointerException(error);
}
//去掉ftp前缀
if (StringUtils.startsWithIgnoreCase(ftpUrl, FTP_PREFIX)) {
ftpUrl = ftpUrl.substring(FTP_PREFIX.length());
}
//去掉path
if (ftpUrl.contains(FTP_PATH_SEPARATOR)) {
ftpUrl = ftpUrl.substring(0, ftpUrl.indexOf(FTP_PATH_SEPARATOR));
}
//获取用户名密码
int hostIndex = ftpUrl.indexOf(FTP_HOST_SEPARATOR);
if (hostIndex >= 0) {
//ftpUrl中包含用户名密码,需要提取
int passIndex = ftpUrl.indexOf(FTP_PASSWORD_SEPARATOR);
if (passIndex > 0 && passIndex < hostIndex) {
String account = ftpUrl.substring(0, hostIndex);
ftpUrl = ftpUrl.substring(hostIndex + 1);
username = account.substring(0, passIndex);
password = account.substring(passIndex + 1);
} else {
error = "ftp的URL格式错误,未提取到登录的用户名和密码.";
logger.error(error);
throw new IllegalArgumentException(error);
}
} else {
//ftpUrl不包含用户名密码,使用匿名登录
username = FTP_ANONYMOUS_NAME;
password = FTP_ANONYMOUS_PASSWORD;
}
//获取端口
int portIndex = ftpUrl.indexOf(FTP_PORT_SEPARATOR);
if (portIndex >= 0) {
//ftpUrl中指定了端口号
port = Integer.parseInt(ftpUrl.substring(portIndex + 1));
} else {
//ftpUrl中未指定端口号,使用默认端口
port = FTP_DEFAULT_PORT;
}
boolean flag;
FTPClient ftpClient = new FTPClient();
try {
ftpClient.connect(ftpUrl, port);
flag = ftpClient.login(username, password);
} catch (IOException e) {
error = "建立FTP连接或者登录出现错误.";
logger.error(error, e);
throw e;
}
if (flag) {
ftpClient.setControlKeepAliveTimeout(FTP_KEEP_ALIVE_TIMEOUT);
ftpClient.setBufferSize(FTP_BUFFER_SIZE);
//尝试进入被动模式
ftpClient.enterLocalPassiveMode();
// 开启服务器对UTF-8的支持,如果服务器支持就用UTF-8编码,否则就使用GBK.
try {
if (!FTPReply.isPositiveCompletion(ftpClient.sendCommand("OPTS UTF8", "ON"))) {
localCharset = ENCODING_GBK;
}
} catch (IOException e) {
localCharset = ENCODING_GBK;
}
return ftpClient;
} else {
error = "登录FTP失败.";
logger.error(error);
throw new RuntimeException(error);
}
}
/**
* 尝试关闭ftp实例
*
* @param ftpClient ftp实例
*/
@Override
public void destroyObject(FTPClient ftpClient) {
try {
if (ftpClient != null && ftpClient.isConnected()) {
ftpClient.logout();
}
} catch (Exception e) {
logger.error("ftp client logout failed...", e);
} finally {
if (ftpClient != null) {
try {
ftpClient.disconnect();
} catch (IOException e) {
logger.error("ftp client disconnect failed...", e);
}
}
}
}
/**
* 验证ftp实例是否可用
*
* @param ftpClient ftp实例
* @return 是否可用
*/
@Override
public boolean validateObject(FTPClient ftpClient) {
try {
if (null != ftpClient && ftpClient.isConnected()) {
return ftpClient.sendNoOp();
}
} catch (Exception e) {
logger.error("Failed to validate client: {}", e);
}
return false;
}
/**
* 激活一个实例(ftp连接池不支持该方法)
*
* @param obj ftp实例
* @throws Exception
*/
@Override
public void activateObject(FTPClient obj) throws Exception {
//Do nothing
throw new UnsupportedOperationException();
}
/**
* 反激活一个实例(ftp连接池不支持该方法)
*
* @param obj ftp实例
* @throws Exception
*/
@Override
public void passivateObject(FTPClient obj) throws Exception {
//Do nothing
throw new UnsupportedOperationException();
}
}
}
文件操作
FileUtil.java
package com;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.net.ftp.FTPClient;
import org.apache.commons.net.ftp.FTPFile;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
/**
* Created by zhy.li on 16/9/22.
*/
public class FileUtil {
private static Logger logger = LoggerFactory.getLogger(FileUtil.class);
private FileUtil() {
}
/**
* 如果文件或目录存在,则删除它们
*
* @param file 文件句柄
* @return 是否删除成功
*/
public static boolean removeFiles(File file) {
try {
if (null == file) {
logger.error("参数错误,待删除的文件句柄为null.");
return false;
}
if (!file.exists()) {
logger.info("文件不存在,无法删除.");
return false;
}
boolean success;
if (file.isFile()) {
//待删除的为文件:直接删除
success = file.delete();
} else if (file.isDirectory()) {
//待删除的为文件夹:先递归删除目录内的子文件/目录,然后删除本目录
File[] files = file.listFiles();
if (null == files) {
logger.error("在获取删除目录内的子文件/目录信息时出现错误.");
return false;
}
for (File sonFile : files) {
if (".".equals(sonFile.getName()) || "..".equals(sonFile.getName())) {
continue;
}
boolean rst = removeFiles(sonFile);
if (!rst) return false;
}
success = file.delete();
} else {
logger.error("待删除文件的类型识别错误.");
return false;
}
return success;
} catch (Exception e) {
logger.error("执行删除文件的过程中,出现异常.", e);
return false;
}
}
/**
* 获取给出的路径的最底层的文件夹路径的集合,以方便按照最底层文件夹级别进行多线程处理
*
* @param file 文件夹路径
* @return 最底层的文件夹路径的集合
*/
public static List<String> getBottomDirs(File file) {
List<String> result = new ArrayList<String>();
if (null == file) {
logger.error("参数错误,文件句柄为null.");
return result;
}
try {
if (!file.isDirectory()) return result;
File[] files = file.listFiles();
if (null == files) return result;
for (File sonFile : files) {
if (".".equals(sonFile.getName()) || "..".equals(sonFile.getName())) {
continue;
}
List<String> sonRst = getBottomDirs(sonFile);
if (!sonRst.isEmpty()) {
//合并子文件夹结果
for (String rst : sonRst) {
result.add(rst);
}
}
}
if (result.isEmpty()) {
//本身就是最底层文件夹
result.add(file.getCanonicalPath());
}
return result;
} catch (Exception e) {
logger.error("获取最底层文件夹路径的集合的过程中,出现异常.");
return new ArrayList<String>();
}
}
/**
* 获取给出的路径的最底层的文件和文件夹路径的集合,以方便按照最底层文件和文件夹级别进行多线程处理
*
* @param file 文件夹路径
* @return 最底层的文件和文件夹路径的集合
*/
public static List<String> getBottomFiles(File file) {
List<String> result = new ArrayList<String>();
if (null == file) {
logger.error("参数错误,文件句柄为null.");
return result;
}
try {
if (!file.exists()) return result;
if (file.isFile()) {
//已经是最底层文件
result.add(file.getCanonicalPath());
return result;
}
File[] files = file.listFiles();
if (null == files) return result;
for (File sonFile : files) {
if (".".equals(sonFile.getName()) || "..".equals(sonFile.getName())) {
continue;
}
List<String> sonRst = getBottomFiles(sonFile);
if (!sonRst.isEmpty()) {
//合并子文件夹结果
for (String rst : sonRst) {
result.add(rst);
}
}
}
if (result.isEmpty()) {
//本身就是最底层文件夹(该文件夹已经没有子文件和子文件夹)
result.add(file.getCanonicalPath());
}
return result;
} catch (Exception e) {
logger.error("获取最底层文件和文件夹路径的集合的过程中,出现异常.");
return new ArrayList<String>();
}
}
/**
* 获取输入的文件和文件夹集合的上一层目录,注意:不验证输入的路径是否存在
*
* @param lowerFiles 某一层的文件和文件夹路径集合
* @return 上一层目录路径的集合
*/
public static List<String> getUpperDirs(List<String> lowerFiles, String separator) {
List<String> result = new ArrayList<String>();
if (null == lowerFiles || lowerFiles.isEmpty()) return result;
for (String lowerFile : lowerFiles) {
//去除末尾的/符号
lowerFile = trimEndSeparator(lowerFile);
String upperDir = lowerFile.substring(0, lowerFile.lastIndexOf(separator));
if (!result.contains(upperDir) && !"".equals(upperDir)) {
result.add(upperDir);
}
}
return result;
}
private static String trimEndSeparator(String oriPath) {
if (null == oriPath) return null;
while ((oriPath.endsWith(File.separator) || oriPath.endsWith(FTPUtil.FTP_PATH_SEPARATOR)) && oriPath.length() > 1) {
oriPath = oriPath.substring(0, oriPath.length() - 1);
}
return oriPath;
}
public static void main(String[] args) {
}
}
Json操作
jsonUtil.java
package com;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.serializer.SerializeConfig;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.google.common.collect.Sets;
import com.qunar.base.qunit.ex.constants.IgnoreDate;
import com.qunar.base.qunit.fastjson.QunitDoubleSerializer;
import com.qunar.base.qunit.response.Response;
import com.qunar.base.qunit.util.PropertyUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
public class JsonUtil {
private static Logger logger = LoggerFactory.getLogger(JsonUtil.class);
private static final String KEY_PREFIX_SEPERATOR = ".";
private static final String AT_MAGIC_FLAG = "__at_magic_flag_411411__";
// Json to Map
@Deprecated
private static List<Map<String, Object>> json2List(Object json) {
JSONArray jsonArr = (JSONArray) json;
List<Map<String, Object>> arrList = new ArrayList<Map<String, Object>>();
for (int i = 0; i < jsonArr.size(); ++i) {
arrList.add(strJson2Map(jsonArr.getString(i)));
}
return arrList;
}
@Deprecated
public static Map<String, Object> strJson2Map(String json) {
JSONObject jsonObject = JSONObject.parseObject(json);
Map<String, Object> resMap = new HashMap<String, Object>();
Iterator<Map.Entry<String, Object>> it = jsonObject.entrySet().iterator();
while (it.hasNext()) {
Map.Entry<String, Object> param = (Map.Entry<String, Object>) it.next();
if (param.getValue() instanceof JSONObject) {
resMap.put(param.getKey(), strJson2Map(param.getValue().toString()));
} else if (param.getValue() instanceof JSONArray) {
resMap.put(param.getKey(), json2List(param.getValue()));
} else {
resMap.put(param.getKey(), JSONObject.toJSONString(param.getValue(), SerializerFeature.WriteClassName));
}
}
return resMap;
}
public static Map<String, Object> json2Map(String json) {
logger.debug("开始将json串转换为Map.");
Map<String, Object> rst = new HashMap<String, Object>();
Object source;
if (StringUtils.isBlank(json)) {
logger.debug("json串为空串,返回空结果");
return rst;
}
//预处理,防止@形式的键值被转换为对象,在转换为map时再替换回来
json = json.replaceAll("@", AT_MAGIC_FLAG);
try {
source = JSON.parse(json);
} catch (Exception e) {
//json不是Json字符串
logger.debug("解析成json格式失败,返回null.待解析的Json内容为:{}", json);
return null;
}
logger.debug("开始循环递归获取基本值的key前缀Map.");
getKeyPrefixMap(null, rst, source);
logger.debug("json串已经转换为map.");
return rst;
}
private static void getKeyPrefixMap(String keyPrefix, Map<String, Object> result, Object object) {
if (null == result) {
String error = "递归获取入key前缀Map时,传入的保存结果的Map为null.";
logger.error(error);
throw new NullPointerException(error);
}
if (object instanceof JSONObject) {
//object为Json串,循环调用该Json串的所有键值
JSONObject jsonObject = (JSONObject) object;
String nextKeyPrefix = keyPrefix == null ? "" : keyPrefix + KEY_PREFIX_SEPERATOR;
for (String key : jsonObject.keySet()) {
getKeyPrefixMap(nextKeyPrefix + key, result, jsonObject.get(key));
}
} else if (object instanceof JSONArray) {
//object为Json数组,循环调用该Json串的所有
JSONArray jsonArray = (JSONArray) object;
String nextKeyPrefix = keyPrefix == null ? "" : keyPrefix + KEY_PREFIX_SEPERATOR;
for (int i = 0; i < jsonArray.size(); i++) {
getKeyPrefixMap(nextKeyPrefix + "[" + i + "]", result, jsonArray.get(i));
}
} else {
//object为基本值,得到一个结果;首先进行@键值的还原
if (object instanceof String) {
object = ((String) object).replaceAll(AT_MAGIC_FLAG, "@");
}
result.put(keyPrefix.replaceAll(AT_MAGIC_FLAG, "@"), object);
}
}
public static Set<String> diffMap(Map<String, Object> source, Map<String, Object> target, String dateIgnore) {
Set<String> result = Sets.newHashSet();
//IgnoreDate type = IgnoreDate.getIgnoreType(ignoreType);
DateProcessor dateProcessor = new DateProcessor();
// todo: fixme:该变量可以下放到对类型type=VALUE处理时
//List<String> datePattern = dateProcessor.getDatePattern(ignoreType);
for (String key : source.keySet()) {
if (target.containsKey(key)) {
//target存在这个key,进行比较值
Object s = source.get(key);
Object t = target.get(key);
// 注意:生成ignore和做全量assert均使用该函数,此处目的为将需要日期忽略的key加入忽略列表.
// 会造成在assert时被直接认为是不同的结果,之所以现在不会出问题,是因为assert比较前已经剔除了忽略的key
if (t != null && !IgnoreDate.NULL.equals(IgnoreDate.getIgnoreType(dateIgnore))) {
if (dateProcessor.isDate(dateIgnore, t.toString())) {
result.add(key);
continue;
}
} /*else if (t != null && type.equals(IgnoreDate.VALUE)) {
for (String pattern : datePattern) {
if (dateProcessor.fuzzyMatch(key, pattern) || dateProcessor.isDate(ignoreType,
t.toString())) {
result.add(key);
continue;
}
}
}*/
if (!isObjectEqual(s, t)) {
result.add(key);
}
} else {
//target不存在这个key
result.add(key);
}
}
for (String key : target.keySet()) {
if (!source.containsKey(key)) {
//else: 会在上面的循环中进行处理
result.add(key);
}
}
return result;
}
public static Set<String> diffMapWithDisorderArray(Map<String, Object> source, Map<String, Object> target, String dateIgnore) {
Set<String> result = Sets.newHashSet();
Set<String> equalRst = Sets.newHashSet(); //空间换时间,记录相等的结果用于全量比较时提效
//IgnoreDate type = IgnoreDate.getIgnoreType(ignoreType);
DateProcessor dateProcessor = new DateProcessor();
// todo: fixme:该变量可以下放到对类型type=VALUE处理时
//List<String> datePattern = dateProcessor.getDatePattern(ignoreType);
for (String key : source.keySet()) {
if (target.containsKey(key)) {
//target存在这个key,进行比较值
Object s = source.get(key);
Object t = target.get(key);
// 注意:生成ignore和做全量assert均使用该函数,此处目的为将需要日期忽略的key加入忽略列表.
// 会造成在assert时被直接认为是不同的结果,之所以现在不会出问题,是因为assert比较前已经剔除了忽略的key
if (t != null && !IgnoreDate.NULL.equals(IgnoreDate.getIgnoreType(dateIgnore))) {
if (dateProcessor.isDate(dateIgnore, t.toString())) {
result.add(key);
continue;
}
} /*else if (t != null && type.equals(IgnoreDate.VALUE)) {
for (String pattern : datePattern) {
if (dateProcessor.fuzzyMatch(key, pattern) || dateProcessor.isDate(ignoreType,
t.toString())) {
result.add(key);
continue;
}
}
}*/
// 数组无序比较之前先同位置比较,在数组基本有序时可以提高效率
if (isObjectEqual(s, t)) {
equalRst.add(key);
continue;
}
// 进行数组无序比较
if (!disorderArrayAssert("", key, source.get(key), target)) {
result.add(key);
} else {
equalRst.add(key);
}
} else {
//target不存在这个key
result.add(key);
}
}
// 全量比较,现在只需要找出在target中有,但在source中不存在的那些key
for (String key : target.keySet()) {
if (!source.containsKey(key) && !equalRst.contains(key)) {
result.add(key);
}
}
return result;
}
private static boolean isObjectEqual(Object s, Object t) {
if (null == s && null == t) {
//2个key都为null
return true;
}
if ((null == s) || (null == t)) {
//只有1个key为null,另1个不为null
return false;
}
//因为sonar检查而注释
// if (s == t) {
// //是同一个对象
// continue;
// }
if (s.equals(t)) {
//两个对象"相等"
return true;
}
if (s.getClass() == t.getClass() && s.toString().equals(t.toString())) {
//两个对象的类型相同,且转换为字符串后比较相同
return true;
}
//非以上情况,视为key的类型不同或者值不同
return false;
}
private static boolean disorderArrayAssert(String keyPrefix, String expKey, Object expValue, Map<String, Object> resultAct) {
int arrayBegin = expKey.indexOf("[");
if (arrayBegin < 0) {
//递归出口:不含有数组, 直接进行比较即可
String fullKey = (StringUtils.isBlank(keyPrefix) ? "" : keyPrefix) + expKey;
if (resultAct.containsKey(fullKey)) {
return isObjectEqual(expValue, resultAct.get(fullKey));
} else {
return false;
}
} else {
// 将最上层的数组循环进行处理, 递归调用求解
int arrayEnd = expKey.indexOf("]");
if (arrayEnd <= arrayBegin) {
logger.error("Error: 字符串格式解析错误,未找到匹配的数组下标,字符串为: {}", expKey);
return false;
}
String arrayPrefix = (StringUtils.isBlank(keyPrefix) ? "" : keyPrefix) + expKey.substring(0, arrayBegin + 1);
String arrayPostfix = expKey.substring(arrayEnd, expKey.length());
int i = 0;
while (true) {
String arrayFull = arrayPrefix + i + arrayPostfix;
if (resultAct.containsKey(arrayFull)) {
//期望中存在对应的数组原因, 递归调用进行比较
if (disorderArrayAssert(arrayPrefix + i + "]", arrayPostfix.substring(1, arrayPostfix.length()), expValue, resultAct)) {
return true;
}
} else {
// 数组越界
return false;
}
i += 1;
}
}
}
public static String response2Json(Response response) {
if (null == response) return null;
//处理body非json的情况
Object body = response.getBody();
Object bodyJson;
try {
if (body instanceof String) {
//字符串被JSON.toJSON()设定为基本类型,不再处理而是直接返回
bodyJson = JSON.parse((String) body);
response.setBody(bodyJson);
} else if (!(body instanceof JSON)) {
bodyJson = JSON.toJSON(body);
response.setBody(bodyJson);
}
} catch (Exception e) {
logger.error("尝试将Response的body转变为Json格式时出错,将直接保持body为原格式不变");
}
Boolean jsonWriteOriginalDoubleValue = Boolean.valueOf(PropertyUtils.getProperty("json_write_original_double_value", "false"));
SerializeConfig config = new SerializeConfig();
if (jsonWriteOriginalDoubleValue) {
config.setAsmEnable(false);
config.put(Double.class, QunitDoubleSerializer.INSTANCE);
}
return JSON.toJSONString(response, config, SerializerFeature.WriteMapNullValue);
}
public static void main(String[] args) {
//{"key1": "1111", "key2" : {"key21": 2121, "key22":[221, 222, "223"]}}
}
}
基于DBUnit的数据表比较
/**
* 比较两个数据表,得到不同的数据表或字段
* @param sourceTable 待比较的数据表
* @param targetTable 待比较的数据表
* @return 2个数据表不同之处,格式:1)table1(col1,col2); 表示:table1的col1和col2字段不同; 2)table1; 表示:table1的整个数据表都不同
*/
public static String generateTableDiff(ITable sourceTable, ITable targetTable) {
logger.info("开始进行数据表的比较");
if ((null == sourceTable || null == sourceTable.getTableMetaData()) && (null == targetTable || null == targetTable.getTableMetaData())) {
logger.info("2个数据表的内容均为空,返回结果:无差异");
return "";
}
if (null == sourceTable || null == sourceTable.getTableMetaData()) {
logger.info("1个数据表的内容不为空,另一个数据表的内容为空,返回结果:整个数据表均有差异");
return targetTable.getTableMetaData().getTableName() + ";";
}
if (null == targetTable || null == targetTable.getTableMetaData()) {
logger.info("1个数据表的内容不为空,另一个数据表的内容为空,返回结果:整个数据表均有差异");
return sourceTable.getTableMetaData().getTableName() + ";";
}
ITableMetaData sourceTableMetaData = sourceTable.getTableMetaData();
ITableMetaData targetTableMetaData = targetTable.getTableMetaData();
logger.info("2个数据表均不为空,2个数据表的表名为:{}和{},开始比较数据表的内容.", sourceTableMetaData.getTableName(), targetTableMetaData.getTableName());
if (!StringUtils.equals(sourceTableMetaData.getTableName(), targetTableMetaData.getTableName())) {
logger.info("2个数据表的表名不相同,返回结果:整个数据表均有差异");
return sourceTableMetaData.getTableName() + ";" + targetTableMetaData.getTableName() + ";";
}
//比较行数
if (sourceTable.getRowCount() != targetTable.getRowCount()) {
logger.info("2个数据表的行数不相同,返回结果:整个数据表均有差异");
return sourceTableMetaData.getTableName() + ";";
}
//比较列
Columns.ColumnDiff columnDiff;
Column[] sourceColumns;
Column[] targetColumns;
try {
sourceColumns = Columns.getSortedColumns(sourceTableMetaData);
targetColumns = Columns.getSortedColumns(targetTableMetaData);
columnDiff = Columns.getColumnDiff(sourceTableMetaData, targetTableMetaData);
} catch (DataSetException e) {
logger.info("读取数据表的字段结构出现异常,返回结果:整个数据表均有差异");
return sourceTableMetaData.getTableName() + ";";
}
//更新:改为仅判断期望是否有差异的字段;
// 因为数据库中为null的实际数据在保存到xml文件中时会丢失,造成在比较时,期望字段比实际数据库字段少
//if (columnDiff.hasDifference()) {
if (columnDiff.getExpected().length > 0) {
//一般case执行后不应该有列的变化;如果列有增删改时,直接返回整个数据表忽略
logger.info("2个数据表的字段结构不一致({}),返回结果:整个数据表均有差异", columnDiff.toString());
return sourceTableMetaData.getTableName() + ";";
} else if (columnDiff.getActual().length > 0) {
logger.info("实际数据表的字段比期望数据表的字段多,一般原因为该数据表有字段的实际取值为nul,因此不会记录在xml文件中导致;" +
"因为不会导致assert失败,故继续比较;内容如下:{}.", columnDiff.toString());
}
//比较列的数据类型
for (int j = 0; j < sourceColumns.length; j++) {
Column sourceColumn = sourceColumns[j];
Column targetColumn = targetColumns[j];
DataType sourceDataType = sourceColumn.getDataType();
DataType targetDataType = targetColumn.getDataType();
if (!(sourceDataType instanceof UnknownDataType) && !(targetDataType instanceof UnknownDataType) && !(sourceDataType.getClass().isInstance(targetColumn))) {
//列的数据类型均存在,且不相同;注:实际上从xml读取的字段类型总是为unknown
logger.info("2个数据表的字段结构不一致(字段{}的数据类型不同),返回结果:整个数据表均有差异", sourceColumn.getColumnName());
return sourceTableMetaData.getTableName() + ";";
}
}
//比较数据
int rowCount = sourceTable.getRowCount();
StringBuilder buf = new StringBuilder();
for (Column column : sourceColumns) {
for (int i = 0; i < rowCount; i++) {
Object sourceValue;
Object targetValue;
try {
sourceValue = sourceTable.getValue(i, column.getColumnName());
} catch (DataSetException e) {
//异常为该行数据无该列字段
sourceValue = null;
}
try {
targetValue = targetTable.getValue(i, column.getColumnName());
} catch (DataSetException e) {
//异常为该行数据无该列字段
targetValue = null;
}
if (!StringUtils.equals(sourceValue != null ? sourceValue.toString() : null, targetValue != null ? targetValue.toString() : null)) {
//发现不同数据,记录该列为diff字段,并无须再比较剩下的行
logger.info("2个数据表的字段:{}出现不同数据,进行记录并继续下个字段的检查.", column.getColumnName());
buf.append(",").append(column.getColumnName());
break;
}
}
}
String diffRst = buf.toString();
if (StringUtils.isNotBlank(diffRst)) {
logger.info("2个数据表的所有字段已经全部比较完毕,差异结果为:{}", diffRst);
return sourceTableMetaData.getTableName() + "(" + diffRst.substring(1) + ");";
}
logger.info("2个数据表的结构和数据完全一致,返回结果:无差异");
return "";
}
对特定格式字符串的处理
/**
* 将字符串形式的DB表达转换成Map方式
* @param dbStr 表示DB的字符串,格式:db1(table1(col1,col2);table2);db2(table21(co21))
* @return 转换后的Map,key为db名称,value为数据表字符串
*/
private static Map<String, String> dbStr2Map(String dbStr) {
//格式:db1(table1(col1,col2);table2);db2(table21(co21))
Map<String, String> dbMap = new HashMap<String, String>();
if (StringUtils.isBlank(dbStr)) return dbMap;
int depth = 0;
String dbName = "";
String tableStr = "";
//以char形式遍历字符串每个字符,解析字符串为Map
for (int i = 0; i < dbStr.length(); i++) {
char c = dbStr.charAt(i);
switch (c) {
case '(':
if (0 != depth) {
tableStr += c;
}
depth++;
break;
case ')':
depth--;
if (0 > depth) {
//左圆括号和右圆括号个数不一致
throw new RuntimeException("输入的DB字符串的格式错误:左圆括号和右圆括号个数不一致");
}
if (0 == depth) {
//又回到顶层,完成了遍历一个DB项,进行记录;并初始化新的遍历
if (StringUtils.isNotBlank(dbName)) {
dbMap.put(dbName, tableStr);
dbName = "";
tableStr = "";
}
} else {
tableStr += c;
}
break;
case ';':
if (0 != depth) {
tableStr += c;
}
break;
default:
if (0 == depth) {
dbName += c;
} else {
tableStr += c;
}
}
}
if (0 != depth) {
//左圆括号和右圆括号个数不一致
throw new RuntimeException("输入的DB字符串的格式错误:左圆括号和右圆括号个数不一致");
}
return dbMap;
}
/**
* 将数据表描述从字符串形式转换为Map形式
*
* @param input 字符串形式: A(a1,a2);B;C(c1)
* @return Map形式:{A=[a1,a2],B=null,C=[c1]}
*/
public static Map<String, List<String>> tablesStr2Map(String input) {
final String SEPARATOR1 = ";"; //key之间的分割符
final String SEPARATOR2 = ","; //value之间的分割符
Map<String, List<String>> result = new HashMap<String, List<String>>();
if (StringUtils.isBlank(input)) return result;
String[] tables = StringUtils.split(input, SEPARATOR1);
for (String table : tables) {
String temp = StringUtils.trim(table);
if (StringUtils.isBlank(temp)) continue;
if (temp.contains("(") && temp.endsWith(")")) {
int index = temp.indexOf("(");
String tableName = temp.substring(0, index);
if (result.containsKey(tableName) && null == result.get(tableName)) {
//此情况说明result表里已经有忽略整个数据表的表名,不应该再添加忽略的字段
continue;
}
String columnStr = temp.substring(index + 1, temp.length() - 1);
String[] columns = StringUtils.split(columnStr, SEPARATOR2);
List<String> columnList = result.get(tableName);
if (columnList == null) {
columnList = new ArrayList<String>();
result.put(tableName, columnList);
}
columnList.addAll(Arrays.asList(columns));
} else {
//if (!result.containsKey(temp)) {
result.put(temp, null);
//}
}
}
//对map的value进行trim()和去重
removeDuplicate(result);
return result;
}
/**
* 将数据表表述从Map形式转换为字符串形式
*
* @param dbMap Map形式:A={[a1,a2],B=null,C=[c1]}
* @return 字符串形式: A(a1,a2);B;C(c1)
*/
public static String tablesMap2Str(Map<String, List<String>> dbMap) {
final String SEPARATOR1 = ";"; //key之间的分割符
final String SEPARATOR2 = ","; //value之间的分割符
if (null == dbMap || dbMap.isEmpty()) return "";
//对map的value进行trim()和去重
removeDuplicate(dbMap);
String result = "";
for (String dbName : dbMap.keySet()) {
if (null != dbName) {
if (null != dbMap.get(dbName) && !dbMap.get(dbName).isEmpty()) {
result += dbName + "(";
List<String> tables = dbMap.get(dbName);
StringBuilder buf = new StringBuilder();
for (String table : tables) {
buf.append(table).append(SEPARATOR2);
}
result += buf.toString();
result = result.substring(0, result.length() - 1);
result += ")" + SEPARATOR1;
} else {
result += dbName + SEPARATOR1;
}
}
}
//去除最后的分号
if (result.endsWith(SEPARATOR1)) {
result = result.substring(0, result.length() - 1);
}
return result;
}
/**
* 1.对map的值进行trim()
* 2.对map的值进行去重
*
* @param dbMap 去重后map
*/
private static void removeDuplicate(Map<String, List<String>> dbMap) {
if (null == dbMap) return;
for (String key : dbMap.keySet()) {
List<String> value = dbMap.get(key);
if (null == value) continue;
Set<String> setValue = new HashSet<String>();
for (int i = 0; i < value.size(); i++) {
String e = value.get(i);
//先做简化处理
if (null != e) {
e = e.replaceAll("\n", "").replaceAll("\r", "").replaceAll("\t", "").trim();
}
if (setValue.contains(e)) {
//该元素已经存在
value.remove(i);
i--;
} else {
//该元素还不存在
setValue.add(e);
value.set(i, e);
}
}
}
}