1113. Integer Set Partition 解析

本文介绍了一种通过排序和从两端向中间累加元素值的方法来平衡划分数组的算法。该算法适用于需要将数组划分为两个子集的问题,使得这两个子集在元素数量和总和上尽可能接近。

排序然后两头开始加,双数一边一半就好,单数看给左边还是右边。

#include <iostream>
#include <vector>
#include <algorithm>

using namespace std;

int n;
vector <int> list;


int main() {
	cin >> n;

	int temp;
	for (int i = 0; i < n; i++) {
		scanf("%d",&temp);
		list.push_back(temp);
	}

	sort(list.begin(), list.end());

#ifdef _DEBUG
	for (int i = 0; i < list.size(); i++) {
		cout << list[i] << endl;
	}
#endif

	//处理分组
	int s1 = 0, s2 = 0;
	int n1 = 0, n2 = 0;
	int p1 = 0, p2 = list.size() - 1;
	while (p1 < p2) {
		s1 += list[p1];
		s2 += list[p2];
		n1++, n2++, p1++, p2--;
	}

	if (p1 == p2) {//奇数情况
		int ts1 = s1 + list[p1], ts2 = s2 + list[p2];
		if (abs(ts1 - s2) > abs(ts2 - s1)) {
			s1 = ts1, n1++;
		}
		else {
			s2 = ts2, n2++;
		}
	}

	cout << abs(n1 - n2) << " " << abs(s1 - s2) << endl;

	return 0;

}


// // Source code recreated from a .class file by IntelliJ IDEA // (powered by FernFlower decompiler) // package com.tplink.smb.eventcenter.port.local; import com.tplink.smb.eventcenter.api.BaseEvent; import com.tplink.smb.eventcenter.api.Event; import com.tplink.smb.eventcenter.api.EventCenter; import com.tplink.smb.eventcenter.api.EventFuture; import com.tplink.smb.eventcenter.api.EventHandler; import com.tplink.smb.eventcenter.api.EventV2; import com.tplink.smb.eventcenter.api.GenericEventFuture; import com.tplink.smb.eventcenter.api.GenericEventHandler; import com.tplink.smb.eventcenter.api.constant.EventCenterSendResult; import com.tplink.smb.eventcenter.api.constant.GenericEventCenterSendResult; import com.tplink.smb.eventcenter.api.constant.PartitionAssignorMode; import com.tplink.smb.eventcenter.api.constant.SerializeEnum; import com.tplink.smb.eventcenter.api.exception.DuplicateGroupIdException; import com.tplink.smb.eventcenter.core.DataProcessor; import com.tplink.smb.eventcenter.core.GenericDataProcessor; import java.util.Collection; import java.util.HashSet; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicReference; import javax.annotation.Nonnull; import javax.annotation.PreDestroy; import lombok.NonNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.util.CollectionUtils; public class LocalEventCenter implements EventCenter { private static final Logger log = LoggerFactory.getLogger(LocalEventCenter.class); private final ConcurrentHashMap<String, AtomicReference<Set<String>>> topicGroupMap = new ConcurrentHashMap(4); private final ConcurrentHashMap<String, EventHandler> groupHandlerMap = new ConcurrentHashMap(4); private final ConcurrentHashMap<String, ExecutorService> groupExecutorMap = new ConcurrentHashMap(4); private final ConcurrentHashMap<String, AtomicReference<Set<String>>> genericTopicGroupMap = new ConcurrentHashMap(4); private final ConcurrentHashMap<String, GenericEventHandler<?>> genericGroupHandlerMap = new ConcurrentHashMap(4); private final ConcurrentHashMap<String, ExecutorService> genericGroupExecutorMap = new ConcurrentHashMap(4); private final String METHOD_NOT_IMPLEMENT = "Method not implement yet, use send instead."; public LocalEventCenter() { } public void send(@Nonnull String topic, @Nonnull Event event) { this.send(topic, (String)null, (Integer)null, (Event)event, (EventFuture)null); } public void send(@Nonnull String topic, @Nonnull Event event, EventFuture eventFuture) { this.send(topic, (String)null, (Integer)null, (Event)event, (EventFuture)eventFuture); } public void send(@Nonnull String topic, String key, @Nonnull Event event) { this.send(topic, (String)key, (Integer)null, (Event)event, (EventFuture)null); } public void send(@Nonnull String topic, @Nonnull Integer partition, @Nonnull Event event) { this.send(topic, (String)null, (Integer)partition, (Event)event, (EventFuture)null); } public void send(@Nonnull String topic, @Nonnull Integer partition, @Nonnull Event event, EventFuture eventFuture) { this.send(topic, (String)null, (Integer)partition, (Event)event, (EventFuture)eventFuture); } public void send(@Nonnull String topic, String key, @Nonnull Event event, EventFuture eventFuture) { this.send(topic, (String)key, (Integer)null, (Event)event, (EventFuture)eventFuture); } private void send(@Nonnull String topic, String key, Integer partition, @Nonnull Event event, EventFuture eventFuture) { event.setTimeStamp(System.currentTimeMillis()); if (log.isTraceEnabled()) { log.trace("before handle event: {}", event); } if (!this.dispatchEvent(topic, event)) { log.warn("Failed to knock event, filterKey: {}, topic:{}", event.getFilterKey(), topic); } else { if (Objects.nonNull(eventFuture)) { EventCenterSendResult eventCenterSendResult = new EventCenterSendResult(topic, key, event, System.currentTimeMillis(), partition); eventFuture.onSuccess(eventCenterSendResult); } if (log.isTraceEnabled()) { log.trace("after handle event: {}", event); } } } public void registerBroadcast(@Nonnull String topic, @Nonnull EventHandler handler, @Nonnull ExecutorService executorService) { String groupId = UUID.randomUUID().toString(); this.registerUnicast(topic, groupId, handler, executorService); } public void registerBroadcast(@Nonnull String topic, @Nonnull EventHandler handler, @Nonnull ExecutorService executorService, PartitionAssignorMode partitionAssignorMode) { this.registerBroadcast(topic, handler, executorService); } public void registerUnicast(@Nonnull String topic, @Nonnull String groupId, @Nonnull EventHandler handler, @Nonnull ExecutorService executorService) { this.register(topic, groupId, handler, executorService); } public void registerUnicast(@Nonnull String topic, @Nonnull String groupId, @Nonnull EventHandler handler, @Nonnull ExecutorService executorService, PartitionAssignorMode partitionAssignorMode) { this.registerUnicast(topic, groupId, handler, executorService); } public void registerBroadcastWithoutThreadPool(@Nonnull String topic, @Nonnull EventHandler handler, PartitionAssignorMode partitionAssignorMode) { String groupId = UUID.randomUUID().toString(); this.registerUnicastWithoutThreadPool(topic, groupId, handler, partitionAssignorMode); } public void registerUnicastWithoutThreadPool(@Nonnull String topic, @Nonnull String groupId, @Nonnull EventHandler handler, PartitionAssignorMode partitionAssignorMode) { this.register(topic, groupId, handler, (ExecutorService)null); } public void unregister(@Nonnull String topic, @Nonnull EventHandler handler) { boolean haveHandler = false; String groupId = null; for(Map.Entry<String, EventHandler> entry : this.groupHandlerMap.entrySet()) { EventHandler eventHandler = (EventHandler)entry.getValue(); if (Objects.equals(eventHandler, handler)) { groupId = (String)entry.getKey(); haveHandler = true; break; } } if (haveHandler) { this.groupHandlerMap.remove(groupId); this.groupExecutorMap.remove(groupId); AtomicReference<Set<String>> atomicReference = (AtomicReference)this.topicGroupMap.get(topic); this.doUnRegister(groupId, atomicReference); } else { log.warn("No such handler could be found to unregister!"); } } public <T> void sendGenerically(@Nonnull String topic, @Nonnull T event) { this.sendGenerically(topic, (String)null, (Integer)null, event, (GenericEventFuture)null); } public <T> void sendGenerically(@Nonnull String topic, @Nonnull T event, GenericEventFuture<T> genericEventFuture) { this.sendGenerically(topic, (String)null, (Integer)null, event, genericEventFuture); } public <T> void sendGenerically(@Nonnull String topic, String key, @Nonnull T event) { this.sendGenerically(topic, key, (Integer)null, event, (GenericEventFuture)null); } public <T> void sendGenerically(@Nonnull String topic, @Nonnull Integer partition, @Nonnull T event) { this.sendGenerically(topic, (String)null, partition, event, (GenericEventFuture)null); } public <T> void sendGenerically(@Nonnull String topic, @Nonnull Integer partition, @Nonnull T event, GenericEventFuture<T> genericEventFuture) { this.sendGenerically(topic, (String)null, partition, event, genericEventFuture); } public <T> void sendGenerically(@Nonnull String topic, String key, @Nonnull T event, GenericEventFuture<T> genericEventFuture) { this.sendGenerically(topic, key, (Integer)null, event, genericEventFuture); } private <T> void sendGenerically(@Nonnull String topic, String key, Integer partition, @Nonnull T event, GenericEventFuture<T> genericEventFuture) { if (log.isTraceEnabled()) { log.trace("before handle event: {}", event); } if (!this.dispatchEvent(topic, event)) { log.warn("Failed to knock event, topic: {}, key:{}", topic, key); } else { if (Objects.nonNull(genericEventFuture)) { GenericEventCenterSendResult<T> genericEventCenterSendResult = new GenericEventCenterSendResult(topic, key, event, System.currentTimeMillis(), partition); genericEventFuture.onSuccess(genericEventCenterSendResult); } if (log.isTraceEnabled()) { log.trace("after handle event: {}", event); } } } public <T> void registerBroadcastGenerically(@Nonnull String topic, @Nonnull GenericEventHandler<T> handler, @Nonnull ExecutorService executorService, @NonNull Class<T> clazz) { if (clazz == null) { throw new NullPointerException("clazz is marked non-null but is null"); } else { String groupId = UUID.randomUUID().toString(); this.registerUnicastGenerically(topic, groupId, handler, executorService, clazz); } } public <T> void registerBroadcastGenerically(@Nonnull String topic, @Nonnull GenericEventHandler<T> handler, @Nonnull ExecutorService executorService, PartitionAssignorMode partitionAssignorMode, @NonNull Class<T> clazz) { if (clazz == null) { throw new NullPointerException("clazz is marked non-null but is null"); } else { this.registerBroadcastGenerically(topic, handler, executorService, clazz); } } public <T> void registerUnicastGenerically(@Nonnull String topic, @Nonnull String groupId, @Nonnull GenericEventHandler<T> handler, @Nonnull ExecutorService executorService, @NonNull Class<T> clazz) { if (clazz == null) { throw new NullPointerException("clazz is marked non-null but is null"); } else { this.register(topic, groupId, handler, executorService); } } public <T> void registerUnicastGenerically(@Nonnull String topic, @Nonnull String groupId, @Nonnull GenericEventHandler<T> handler, @Nonnull ExecutorService executorService, PartitionAssignorMode partitionAssignorMode, @NonNull Class<T> clazz) { if (clazz == null) { throw new NullPointerException("clazz is marked non-null but is null"); } else { this.registerUnicastGenerically(topic, groupId, handler, executorService, clazz); } } public <T> void registerBroadcastWithoutThreadPoolGenerically(@Nonnull String topic, @Nonnull GenericEventHandler<T> handler, PartitionAssignorMode partitionAssignorMode, @NonNull Class<T> clazz) { if (clazz == null) { throw new NullPointerException("clazz is marked non-null but is null"); } else { String groupId = UUID.randomUUID().toString(); this.registerUnicastWithoutThreadPoolGenerically(topic, groupId, handler, partitionAssignorMode, clazz); } } public <T> void registerUnicastWithoutThreadPoolGenerically(@Nonnull String topic, @Nonnull String groupId, @Nonnull GenericEventHandler<T> handler, PartitionAssignorMode partitionAssignorMode, @NonNull Class<T> clazz) { if (clazz == null) { throw new NullPointerException("clazz is marked non-null but is null"); } else { this.register(topic, groupId, (GenericEventHandler)handler, (ExecutorService)null); } } public <T> void unregisterGenerically(@Nonnull String topic, @Nonnull GenericEventHandler<T> handler) { boolean haveHandler = false; String groupId = null; for(Map.Entry<String, GenericEventHandler<?>> entry : this.genericGroupHandlerMap.entrySet()) { GenericEventHandler<?> eventHandler = (GenericEventHandler)entry.getValue(); if (Objects.equals(eventHandler, handler)) { groupId = (String)entry.getKey(); haveHandler = true; break; } } if (haveHandler) { this.genericGroupHandlerMap.remove(groupId); this.genericGroupExecutorMap.remove(groupId); AtomicReference<Set<String>> atomicReference = (AtomicReference)this.genericTopicGroupMap.get(topic); this.doUnRegister(groupId, atomicReference); } else { log.warn("No such handler could be found to unregister!"); } } public <T> void unregisterGenerically(@Nonnull String topic, @Nonnull GenericEventHandler<T> handler, boolean isDestroyExecutorService) { this.unregisterGenerically(topic, handler); } public void unregister(@Nonnull String topic, @Nonnull EventHandler eventHandler, boolean isDestroyExecutorService) { this.unregister(topic, eventHandler); } public void unregisterAllTopics() { this.destroy(); } @PreDestroy public void destroy() { this.topicGroupMap.clear(); this.groupHandlerMap.clear(); this.groupExecutorMap.clear(); this.genericTopicGroupMap.clear(); this.genericGroupHandlerMap.clear(); this.genericGroupExecutorMap.clear(); } private void doUnRegister(String groupId, AtomicReference<Set<String>> atomicReference) { boolean isUpdate; do { Set<String> oldGroups = (Set)atomicReference.get(); Set<String> newGroups; if (CollectionUtils.isEmpty((Collection)atomicReference.get())) { newGroups = new HashSet(); } else { newGroups = new HashSet(oldGroups); } newGroups.remove(groupId); isUpdate = atomicReference.compareAndSet(oldGroups, newGroups); } while(!isUpdate); } private boolean dispatchEvent(String topic, Event event) { if (!this.topicGroupMap.containsKey(topic)) { log.warn("no handlers for topic: {},", topic); return false; } else { AtomicReference<Set<String>> atomicReference = (AtomicReference)this.topicGroupMap.get(topic); if (!Objects.isNull(atomicReference) && !CollectionUtils.isEmpty((Collection)atomicReference.get())) { for(String groupId : (Set)atomicReference.get()) { EventHandler eventHandler = (EventHandler)this.groupHandlerMap.get(groupId); ExecutorService executorService = (ExecutorService)this.groupExecutorMap.get(groupId); log.debug("handlers: {},", eventHandler.getClass()); if (Objects.nonNull(executorService)) { executorService.submit(new DataProcessor(event, eventHandler)); } else { this.dataProcessor(event, eventHandler); } } return true; } else { return true; } } } private void register(String topic, String groupId, EventHandler handler, ExecutorService executorService) { try { AtomicReference<Set<String>> atomicReference = this.doRegister(groupId, (AtomicReference)this.topicGroupMap.computeIfAbsent(topic, (k) -> new AtomicReference())); this.topicGroupMap.put(topic, atomicReference); this.groupHandlerMap.put(groupId, handler); this.groupExecutorMap.put(groupId, executorService); } catch (DuplicateGroupIdException var6) { log.warn("The groupId has been registered! Please check the value of groupId:{}.topic:{}", groupId, topic); } } private void dataProcessor(Event event, EventHandler eventHandler) { try { eventHandler.handleEvent(event); } catch (Exception e) { log.error("Fail to process event, filterKey:{}, handler:{}", new Object[]{event.getFilterKey(), eventHandler, e}); } } private <T> boolean dispatchEvent(String topic, T event) { if (!this.genericTopicGroupMap.containsKey(topic)) { log.warn("no handlers for topic: {},", topic); return false; } else { AtomicReference<Set<String>> atomicReference = (AtomicReference)this.genericTopicGroupMap.get(topic); if (!Objects.isNull(atomicReference) && !CollectionUtils.isEmpty((Collection)atomicReference.get())) { for(String groupId : (Set)atomicReference.get()) { GenericEventHandler<T> eventHandler = (GenericEventHandler)this.genericGroupHandlerMap.get(groupId); ExecutorService executorService = (ExecutorService)this.genericGroupExecutorMap.get(groupId); log.debug("handlers: {},", eventHandler.getClass()); if (Objects.nonNull(executorService)) { executorService.submit(new GenericDataProcessor(event, eventHandler)); } else { this.dataProcessor(event, eventHandler); } } return true; } else { return true; } } } private <T> void register(String topic, String groupId, GenericEventHandler<T> handler, ExecutorService executorService) { try { AtomicReference<Set<String>> atomicReference = this.doRegister(groupId, (AtomicReference)this.genericTopicGroupMap.computeIfAbsent(topic, (k) -> new AtomicReference())); this.genericTopicGroupMap.put(topic, atomicReference); this.genericGroupHandlerMap.put(groupId, handler); this.genericGroupExecutorMap.put(groupId, executorService); } catch (DuplicateGroupIdException var6) { log.warn("The groupId has been registered! Please check the value of groupId:{}.topic:{}", groupId, topic); } } private AtomicReference<Set<String>> doRegister(String groupId, AtomicReference<Set<String>> atomicReference) throws DuplicateGroupIdException { boolean isUpdate; do { Set<String> oldGroups = (Set)atomicReference.get(); Set<String> newGroups; if (CollectionUtils.isEmpty(oldGroups)) { newGroups = new HashSet(); } else { if (oldGroups.contains(groupId)) { throw new DuplicateGroupIdException(); } newGroups = new HashSet((Collection)atomicReference.get()); } newGroups.add(groupId); isUpdate = atomicReference.compareAndSet(oldGroups, newGroups); } while(!isUpdate); return atomicReference; } private <T> void dataProcessor(T event, GenericEventHandler<T> eventHandler) { try { eventHandler.handleEvent(event); } catch (Exception e) { log.error("Fail to process event, handler:{}", eventHandler, e); } } public void send(@Nonnull String topic, @Nonnull BaseEvent event, @Nonnull SerializeEnum serializeEnum) { switch (serializeEnum) { case KRYO: this.sendWithKryo(topic, (String)null, (Integer)null, (EventV2)event, (GenericEventFuture)null); break; default: log.debug("Method not implement yet, use send instead."); } } public void send(@Nonnull String topic, @Nonnull BaseEvent event, GenericEventFuture eventFuture, @Nonnull SerializeEnum serializeEnum) { switch (serializeEnum) { case KRYO: this.sendWithKryo(topic, (String)null, (Integer)null, (EventV2)event, eventFuture); break; default: log.debug("Method not implement yet, use send instead."); } } public void send(@Nonnull String topic, String key, @Nonnull BaseEvent event, @Nonnull SerializeEnum serializeEnum) { switch (serializeEnum) { case KRYO: this.sendWithKryo(topic, key, (Integer)null, (EventV2)event, (GenericEventFuture)null); break; default: log.debug("Method not implement yet, use send instead."); } } public void send(@Nonnull String topic, String key, @Nonnull BaseEvent event, GenericEventFuture eventFuture, @Nonnull SerializeEnum serializeEnum) { switch (serializeEnum) { case KRYO: this.sendWithKryo(topic, key, (Integer)null, (EventV2)event, eventFuture); break; default: log.debug("Method not implement yet, use send instead."); } } public void send(@Nonnull String topic, @Nonnull Integer partition, @Nonnull BaseEvent event, @Nonnull SerializeEnum serializeEnum) { switch (serializeEnum) { case KRYO: this.sendWithKryo(topic, (String)null, partition, (EventV2)event, (GenericEventFuture)null); break; default: log.debug("Method not implement yet, use send instead."); } } public void send(@Nonnull String topic, @Nonnull Integer partition, @Nonnull BaseEvent event, GenericEventFuture eventFuture, @Nonnull SerializeEnum serializeEnum) { switch (serializeEnum) { case KRYO: this.sendWithKryo(topic, (String)null, partition, (EventV2)event, eventFuture); break; default: log.debug("Method not implement yet, use send instead."); } } private <T> void sendWithKryo(@Nonnull String topic, String key, Integer partition, @Nonnull EventV2<T> event, GenericEventFuture eventFuture) { event.setTimeStamp(System.currentTimeMillis()); if (log.isTraceEnabled()) { log.trace("before handle event: {}", event); } if (!this.dispatchEvent(topic, (Object)event)) { log.warn("Failed to knock event, filterKey: {}, topic:{}", event.getFilterKey(), topic); } else { if (Objects.nonNull(eventFuture)) { GenericEventCenterSendResult<T> genericEventCenterSendResult = new GenericEventCenterSendResult(topic, key, event.getMessage(), System.currentTimeMillis(), partition); eventFuture.onSuccess(genericEventCenterSendResult); } if (log.isTraceEnabled()) { log.trace("after handle event: {}", event); } } } public void registerBroadcast(@Nonnull String topic, @Nonnull GenericEventHandler handler, @Nonnull ExecutorService executorService, @Nonnull SerializeEnum serializeEnum) { switch (serializeEnum) { case KRYO: String groupId = UUID.randomUUID().toString(); this.registerUnicast(topic, groupId, handler, executorService, serializeEnum); break; default: this.registerBroadcastGenerically(topic, handler, executorService, EventV2.class); } } public void registerBroadcast(@Nonnull String topic, @Nonnull GenericEventHandler handler, @Nonnull ExecutorService executorService, PartitionAssignorMode partitionAssignorMode, @Nonnull SerializeEnum serializeEnum) { switch (serializeEnum) { case KRYO: String groupId = UUID.randomUUID().toString(); this.registerUnicast(topic, groupId, handler, executorService, partitionAssignorMode, serializeEnum); break; default: this.registerBroadcastGenerically(topic, handler, executorService, partitionAssignorMode, EventV2.class); } } public void registerUnicast(@Nonnull String topic, @Nonnull String groupId, @Nonnull GenericEventHandler handler, @Nonnull ExecutorService executorService, @Nonnull SerializeEnum serializeEnum) { switch (serializeEnum) { case KRYO: this.registerUnicast(topic, groupId, handler, executorService, (PartitionAssignorMode)null, serializeEnum); break; default: this.registerUnicastGenerically(topic, groupId, handler, executorService, (PartitionAssignorMode)null, EventV2.class); } } public void registerUnicast(@Nonnull String topic, @Nonnull String groupId, @Nonnull GenericEventHandler handler, @Nonnull ExecutorService executorService, PartitionAssignorMode partitionAssignorMode, @Nonnull SerializeEnum serializeEnum) { switch (serializeEnum) { case KRYO: this.registerUnicastGenerically(topic, groupId, handler, executorService, EventV2.class); break; default: this.registerUnicastGenerically(topic, groupId, handler, executorService, partitionAssignorMode, EventV2.class); } } public void registerBroadcastWithoutThreadPool(@Nonnull String topic, @Nonnull GenericEventHandler handler, PartitionAssignorMode partitionAssignorMode, @Nonnull SerializeEnum serializeEnum) { switch (serializeEnum) { case KRYO: String groupId = UUID.randomUUID().toString(); this.registerUnicastWithoutThreadPoolGenerically(topic, groupId, handler, partitionAssignorMode, EventV2.class); break; default: this.registerBroadcastWithoutThreadPoolGenerically(topic, handler, partitionAssignorMode, EventV2.class); } } public void registerUnicastWithoutThreadPool(@Nonnull String topic, @Nonnull String groupId, @Nonnull GenericEventHandler handler, PartitionAssignorMode partitionAssignorMode, @Nonnull SerializeEnum serializeEnum) { switch (serializeEnum) { case KRYO: this.register(topic, groupId, (GenericEventHandler)handler, (ExecutorService)null); break; default: this.registerUnicastWithoutThreadPoolGenerically(topic, groupId, handler, partitionAssignorMode, EventV2.class); } } } 这是项目中local版本的消息中心的实现,理解一下
11-05
robotframework给spark-shell写测试用例,spark-shell的开启方式为: (1)cd /opt/module/ume/spark/bin (2)./spark-shell --master local --conf spark.hadoop.hive.metastore.uris=thrift://local:19083 spark-shell的建表语句为: import org.apache.iceberg.Schema; import org.apache.iceberg.types.Types; //创建schema val schema = new Schema( Types.NestedField.required(1, "level", Types.StringType.get()), Types.NestedField.required(2, "event_time", Types.TimestampType.withoutZone()), //without Zone Types.NestedField.required(3, "message", Types.StringType.get()), Types.NestedField.required(4, "count", Types.IntegerType.get()) ); import org.apache.iceberg.PartitionSpec; //创建分区 val spec = PartitionSpec.builderFor(schema).hour("event_time").gpHash("level", 2).build(); import org.apache.iceberg.hive.HiveCatalog; //创建catalog val catalog = new HiveCatalog(); catalog.setConf(spark.sparkContext.hadoopConfiguration); val properties = Map("warehouse"->"hdfs://node199:9000/user/spark","uri"->"thrift://localhost:19083"); import collection.JavaConversions._ catalog.initialize("hive",mapAsJavaMap(properties)); // 创建数据库,如果spark中此数据库已经存在则可以忽略 import org.apache.iceberg.catalog.Namespace val namespace = Namespace.of("postgres"); import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap catalog.createNamespace(namespace, ImmutableMap.of()); //创建数据表 import org.apache.iceberg.Table; import org.apache.iceberg.catalog.TableIdentifier; val name = TableIdentifier.of("postgres", "test_ts_without_zone"); val table = catalog.createTable(name, schema, spec); 核心是通过robotframework,通过SSHLibrary库的write语句,要解决robotframework与spark-shell的交互问题
03-15
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值