device public set

本文介绍了一种通过分析广告请求流量来查找公共设备ID的方法。详细讨论了三种不同的技术方案:使用MapReduce、Hive表以及从请求日志中选择所有不同的设备ID。最终通过shell命令实现了目标。

backgroud:  our dvertiser provide on device list of idfa to show ad to  target audience,however none of the  ad shows ,so we want to know how many  public device id  in our traffic request。

to find the public  deviceid,we need to get all device id(idfa/google adid) in one day .

method1: use map reduce on azkaban ,however it failed . 

 

method2: use  hive tables;  insert  the deviceidlist to one table and  join  deviceids . 

 

 

method3: select  all distinct deviceids from request log   and output as a file , about 0.2 billion deviceid list  and file size 6G.

then use shell command just as this : 

grep -F -f a.txt  b.txt    >  public_ids.txt 

then ,we get the public deviceids . 

 

refer:http://blog.youkuaiyun.com/autofei/article/details/6579320

 

转载于:https://www.cnblogs.com/lavin/p/6912244.html

package com.kennysoft.grain.protocol; import lombok.Data; import java.sql.Timestamp; // 订阅的身份证报文样例 @Data public class DentityCard { public Headers getHeaders() { return headers; } public void setHeaders(Headers headers) { this.headers = headers; } public MessageType getMesssgeType() { return messsgeType; } public void setMesssgeType(MessageType messsgeType) { this.messsgeType = messsgeType; } public String getDeviceId() { return deviceId; } public void setDeviceId(String deviceId) { this.deviceId = deviceId; } public Properties getProperties() { return properties; } public void setProperties(Properties properties) { this.properties = properties; } public String getDentityCard() { return dentityCard; } public void setDentityCard(String dentityCard) { this.dentityCard = dentityCard; } public String getUserName() { return userName; } public void setUserName(String userName) { this.userName = userName; } public String getAddress() { return address; } public void setAddress(String address) { this.address = address; } public Timestamp getTimestamp() { return timestamp; } public void setTimestamp(Timestamp timestamp) { this.timestamp = timestamp; } /* { "headers": { "productId": "产品 Id", //产品 Id "deviceName": "身份证读卡器", //设备名称 "orgId": "" //设备机构 }, "messageType": "REPORT_PROPERTY", //上报类型 "messageId": "消息 Id", //消息 Id 唯一 "deviceId": "设备 Id", //设备Id,由云端分配 "properties": { }, "dentityCard": "身份证号", //读取到的身份证号,进行对称加密 "userName": "用户姓名", //用户姓名 "address": "地址", //地址 "timestamp": 1640939096682 } */ @Data public class Headers { public Headers() { productId = ""; deviceName = ""; orgId = ""; } String productId; String deviceName; String orgId; public String getProductId() { return productId; } public void setProductId(String productId) { this.productId = productId; } public String getDeviceName() { return deviceName; } public void setDeviceName(String deviceName) { this.deviceName = deviceName; } public String getOrgId() { return orgId; } public void setOrgId(String orgId) { this.orgId = orgId; } } public enum MessageType { REPORT_PROPERTY } @Data public class Properties { } public DentityCard() { headers = new Headers(); messsgeType = MessageType.REPORT_PROPERTY; deviceId = ""; properties = new Properties(); dentityCard = ""; userName = ""; address = ""; timestamp = new Timestamp(System.currentTimeMillis()); } Headers headers; MessageType messsgeType; String deviceId; Properties properties; String dentityCard; String userName; String address; Timestamp timestamp; }
11-24
package com.kennysoft.grain.protocol; import lombok.Data; import java.sql.Timestamp; //订阅的抓拍数据报文样例 public class Photo { public Timestamp getTimestamp() { return timestamp; } public void setTimestamp(Timestamp timestamp) { this.timestamp = timestamp; } public Properties getProperties() { return properties; } public void setProperties(Properties properties) { this.properties = properties; } public String getDeviceId() { return deviceId; } public void setDeviceId(String deviceId) { this.deviceId = deviceId; } public String getMessageId() { return messageId; } public void setMessageId(String messageId) { this.messageId = messageId; } public MessageType getMesssgeType() { return messsgeType; } public void setMesssgeType(MessageType messsgeType) { this.messsgeType = messsgeType; } public Headers getHeaders() { return headers; } public void setHeaders(Headers headers) { this.headers = headers; } /* { 4.5.5 抓拍报文字段说明 "headers": { "productId": "202109131753", "keepOnlineTimeoutSeconds": 600, "keepOnline": true, "deviceName": "地磅摄像头 1" }, "messageType": "REPORT_PROPERTY", "messageId": "1652074944674", "deviceId": "91330822704614086G001_91330822704614086G0010000_002_004_004_004", "properties": { "carNumber": "川 AH1636", "position": "big" }, "timestamp": 1652074944674 } */ @Data public class Headers { public Headers(){ productId =""; keepOnlineTimeoutSeconds =0; keepOnline =true; deviceName =""; } String productId; int keepOnlineTimeoutSeconds; boolean keepOnline; String deviceName; public String getProductId() { return productId; } public void setProductId(String productId) { this.productId = productId; } public int getKeepOnlineTimeoutSeconds() { return keepOnlineTimeoutSeconds; } public void setKeepOnlineTimeoutSeconds(int keepOnlineTimeoutSeconds) { this.keepOnlineTimeoutSeconds = keepOnlineTimeoutSeconds; } public boolean isKeepOnline() { return keepOnline; } public void setKeepOnline(boolean keepOnline) { this.keepOnline = keepOnline; } public String getDeviceName() { return deviceName; } public void setDeviceName(String deviceName) { this.deviceName = deviceName; } } public enum MessageType { REPORT_PROPERTY } @Data public class Properties { public Properties(){ carNumber =""; position =""; } String carNumber; String position; public String getCarNumber() { return carNumber; } public void setCarNumber(String carNumber) { this.carNumber = carNumber; } public String getPosition() { return position; } public void setPosition(String position) { this.position = position; } } public Photo(){ headers =new Headers(); messsgeType = MessageType.REPORT_PROPERTY; messageId =""; deviceId =""; properties = new Properties(); timestamp = new Timestamp(System.currentTimeMillis()); } Headers headers; MessageType messsgeType; String messageId; String deviceId; Properties properties; Timestamp timestamp; }
11-24
package com.kennysoft.grain.protocol; import lombok.Data; import java.sql.Timestamp; import java.util.List; @Data public class LED { /* parameter: { messageType: 'INVOKE_FUNCTION', inputs: [ { name: 'devId', value: devId, }, { name: 'carNumber', value: carNumber, }, { name: 'grossWeight', value: grossWeight, }, { name: 'tareWeight', value: tareWeight, }, ], }, qos: 0, } { "output": true, //调用结果,true or false "headers": { "productId": "产品Id", //产品Id "deviceName": "led", //设备名称 "orgId": "" //组织机构 }, "messageType": "INVOKE_FUNCTION_REPLY", //消息类型 "success": true, //调用结果,true or false "messageId": "消息Id", //消息Id,唯一 "deviceId": "设备Id", //设备Id, "timestamp": 1640932814641 //时间戳 } */ @Data public class Parameter { List<Input> inputs; int qos; public List<Input> getInputs() { return inputs; } public void setInputs(List<Input> inputs) { this.inputs = inputs; } public int getQos() { return qos; } public void setQos(int qos) { this.qos = qos; } } @Data public class Input { public Input(){ name =""; value =""; } String name; String value; public String getName() { return name; } public void setName(String name) { this.name = name; } public String getValue() { return value; } public void setValue(String value) { this.value = value; } } public class Headers { public Headers() { productId = ""; deviceName = ""; orgId = ""; } String productId; String deviceName; String orgId; public String getProductId() { return productId; } public void setProductId(String productId) { this.productId = productId; } public String getDeviceName() { return deviceName; } public void setDeviceName(String deviceName) { this.deviceName = deviceName; } public String getOrgId() { return orgId; } public void setOrgId(String orgId) { this.orgId = orgId; } } public enum MessageType { INVOKE_FUNCTION_REPLY } public LED(){ messageType = MessageType.INVOKE_FUNCTION_REPLY; success = false; messageId =""; deviceId =""; timestamp = new Timestamp(System.currentTimeMillis()); } MessageType messageType; //消息类型 boolean success; String messageId; String deviceId; Timestamp timestamp; public MessageType getMessageType() { return messageType; } public void setMessageType(MessageType messageType) { this.messageType = messageType; } public boolean isSuccess() { return success; } public void setSuccess(boolean success) { this.success = success; } public String getMessageId() { return messageId; } public void setMessageId(String messageId) { this.messageId = messageId; } public String getDeviceId() { return deviceId; } public void setDeviceId(String deviceId) { this.deviceId = deviceId; } public Timestamp getTimestamp() { return timestamp; } public void setTimestamp(Timestamp timestamp) { this.timestamp = timestamp; } }
11-24
内容概要:本文系统阐述了Java Persistence API(JPA)的核心概念、技术架构、核心组件及实践应用,重点介绍了JPA作为Java官方定义的对象关系映射(ORM)规范,如何通过实体类、EntityManager、JPQL和persistence.xml配置文件实现Java对象与数据库表之间的映射与操作。文章详细说明了JPA解决的传统JDBC开发痛点,如代码冗余、对象映射繁琐、跨数据库兼容性差等问题,并解析了JPA与Hibernate、EclipseLink等实现框架的关系。同时提供了基于Hibernate和MySQL的完整实践案例,涵盖Maven依赖配置、实体类定义、CRUD操作实现等关键步骤,并列举了常用JPA注解及其用途。最后总结了JPA的标准化优势、开发效率提升能力及在Spring生态中的延伸应用。 适合人群:具备一定Java基础,熟悉基本数据库操作,工作1-3年的后端开发人员或正在学习ORM技术的中级开发者。 使用场景及目标:①理解JPA作为ORM规范的核心原理与组件协作机制;②掌握基于JPA+Hibernate进行数据库操作的开发流程;③为技术选型、团队培训或向Spring Data JPA过渡提供理论与实践基础。 阅读建议:此资源以理论结合实践的方式讲解JPA,建议读者在学习过程中同步搭建环境,动手实现文中示例代码,重点关注EntityManager的使用、JPQL语法特点以及注解配置规则,从而深入理解JPA的设计思想与工程价值。
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值