Movatterモバイル変換


[0]ホーム

URL:


Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

Sign up
Appearance settings

Commit74f6d7a

Browse files
zhoujinsongjinsongzhouxiaosefengczy006baiyangtx
authored
[AMORO-2204] Support Spark 3.5 for mixed format tables (#3428)
* add tencent-dlc profile* Add amoro-dlc Dockerfile* Change derby url* Fix amoro-dlc Dockerfile* Fix ams start cmd* Rollback change in entrypoint.sh* Remove lake fs dependencies* Add lake fs dependencies* Change hive dependency to dlc version* Remove hive version in tencent-dlc profile* Simplify amoro-dlc Dockerfile* Copy entrypoint.sh* Fix amoro-dlc Dockerfile* Fix amoro-dlc Dockerfile* Fix amoro-dlc Dockerfile* Add some debug info for amoro-dlc Dockerfile* Fix amoro-dlc Dockerfile* Add optimizer task class loader* Exclude not needed classes from spark optimizer* Rollback unnecessary changes* Solve the problem of orc package conflict and thrift size limit.* Optimize database configuration for mysql5.6.* Modify thrift's maxMessageSize to 1000m.* Full support AMORO_CONF_DIR envrionment variable* Fix a load-config.sh export error* Add conf dir to class path* Fix ams.sh classpath envrionment value error* Change hive version to 2.3.9 for dlc profile* Determine change store by name* Remove not needed list database calls* Allow configure catalog-impl for hive type* Support catalog-impl property for spark mixed-format* Fix checkstyle errors* Optimize catalog creation API, no need to upload files first (merge request !14)Squash merge branch 'dev/xiaosefeng-catalog' into 'tencent-master'Optimize catalog creation API, no need to upload files first* Fixed display error when switching catalog authentication mode* Fix tencent-dlc docker file build error* Modify cos related dependency packages and dependency sources* Rewrite pos delete files not written by optimizing* Fix log4j2 info log rollover issue* Add amoro conf into classpath* init mixed spark 3.5 modules* adapt mixformate for spark3.5* Rollback some unnecessary changes* Rollback unnecessary changes* Rollback unnecessary changes* Fixed some unit test issues* Fixed some unit test issues* Fix some comments---------Co-authored-by: jinsongzhou <jinsongzhou@tencent.com>Co-authored-by: xiaosefeng <xiaosefeng@tencent.com>Co-authored-by: ConradJam <jam.gzczy@gmail.com>Co-authored-by: baiyangtx <xiangnebula@163.com>
1 parent647478e commit74f6d7a

File tree

139 files changed

+45012
-58
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

139 files changed

+45012
-58
lines changed

‎amoro-ams/src/main/java/org/apache/amoro/server/table/internal/InternalTableConstants.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
packageorg.apache.amoro.server.table.internal;
2020

2121
importorg.apache.amoro.mixed.InternalMixedIcebergCatalog;
22+
importorg.apache.amoro.table.MixedTable;
2223

2324
/** Constants defines for internal table */
2425
publicclassInternalTableConstants {
@@ -37,6 +38,5 @@ public class InternalTableConstants {
3738
publicstaticfinalStringOSS_PROTOCOL_PREFIX ="oss://";
3839

3940
publicstaticfinalStringCHANGE_STORE_TABLE_NAME_SUFFIX =
40-
InternalMixedIcebergCatalog.CHANGE_STORE_SEPARATOR
41-
+InternalMixedIcebergCatalog.CHANGE_STORE_NAME;
41+
InternalMixedIcebergCatalog.CHANGE_STORE_SEPARATOR +MixedTable.CHANGE_STORE_IDENTIFIER;
4242
}

‎amoro-ams/src/main/resources/mysql/upgrade-0.4.1-to-0.5.0.sql

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ CREATE TABLE `table_identifier`
5959
`table_name`varchar(128)NOT NULL COMMENT'Table name',
6060
PRIMARY KEY (`table_id`),
6161
UNIQUE KEY`table_name_index` (`catalog_name`,`db_name`,`table_name`)
62-
) ROW_FORMAT=DYNAMIC;
62+
)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT'Table identifier for AMS'ROW_FORMAT=DYNAMIC;
6363
INSERT INTO`table_identifier` (`catalog_name`,`db_name`,`table_name`)SELECT`catalog_name`,`db_name`,`table_name`FROM`table_metadata`;
6464

6565
-- table_metadata

‎amoro-ams/src/test/java/org/apache/amoro/server/util/TestIcebergTableUtil.java

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
importorg.apache.amoro.io.IcebergDataTestHelpers;
2222
importorg.apache.amoro.io.MixedDataTestHelpers;
2323
importorg.apache.amoro.mixed.MixedTables;
24+
importorg.apache.amoro.properties.CatalogMetaProperties;
2425
importorg.apache.amoro.server.utils.IcebergTableUtil;
2526
importorg.apache.amoro.shade.guava32.com.google.common.collect.Lists;
2627
importorg.apache.amoro.shade.guava32.com.google.common.collect.Maps;
@@ -123,7 +124,12 @@ private static Table newIcebergTable(Catalog catalog) {
123124
}
124125

125126
privatestaticMixedTablenewMixedTable(Catalogcatalog,booleanwithKey) {
126-
MixedTablesmixedTables =newMixedTables(TableMetaStore.EMPTY,Maps.newHashMap(),catalog);
127+
MixedTablesmixedTables =
128+
newMixedTables(
129+
TableMetaStore.EMPTY,
130+
Maps.newHashMap(),
131+
catalog,
132+
CatalogMetaProperties.MIXED_FORMAT_TABLE_STORE_SEPARATOR_DEFAULT);
127133
returnmixedTables.createTable(
128134
org.apache.amoro.table.TableIdentifier.of("cata","db","table"),
129135
schema,

‎amoro-common/src/main/java/org/apache/amoro/client/PoolConfig.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ public void setMaxReconnects(int maxReconnects) {
8888

8989
publicstaticPoolConfig<?>forUrl(Stringurl) {
9090
PoolConfig<?>poolConfig =newPoolConfig<>();
91-
URLEncodedUtils.parse(URI.create(url),Charset.defaultCharset())
91+
URLEncodedUtils.parse(URI.create(url),String.valueOf(Charset.defaultCharset()))
9292
.forEach(
9393
pair -> {
9494
try {

‎amoro-common/src/main/java/org/apache/amoro/client/ThriftClientPool.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@ public class ThriftClientPool<
4040
privatestaticfinalLoggerLOG =LoggerFactory.getLogger(ThriftClientPool.class);
4141
privatestaticfinalintRECONNECT_INTERVAL =2000;
4242
privatestaticfinalintBORROW_ATTEMPTS =5;
43-
4443
privatefinalThriftClientFactoryclientFactory;
4544
privatefinalThriftPingFactorypingFactory;
4645
privatefinalGenericObjectPool<ThriftClient<T>>pool;

‎amoro-common/src/main/java/org/apache/amoro/properties/CatalogMetaProperties.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,10 @@ public class CatalogMetaProperties {
2525
publicstaticfinalStringSTORAGE_CONFIGS_KEY_HDFS_SITE ="hadoop.hdfs.site";
2626
publicstaticfinalStringSTORAGE_CONFIGS_KEY_CORE_SITE ="hadoop.core.site";
2727
publicstaticfinalStringSTORAGE_CONFIGS_KEY_HIVE_SITE ="hive.site";
28+
2829
publicstaticfinalStringSTORAGE_CONFIGS_KEY_REGION ="storage.s3.region";
2930
publicstaticfinalStringSTORAGE_CONFIGS_KEY_S3_ENDPOINT ="storage.s3.endpoint";
3031
publicstaticfinalStringSTORAGE_CONFIGS_KEY_OSS_ENDPOINT ="storage.oss.endpoint";
31-
3232
publicstaticfinalStringSTORAGE_CONFIGS_VALUE_TYPE_HDFS_LEGACY ="hdfs";
3333
publicstaticfinalStringSTORAGE_CONFIGS_VALUE_TYPE_HADOOP ="Hadoop";
3434
publicstaticfinalStringSTORAGE_CONFIGS_VALUE_TYPE_S3 ="S3";

‎amoro-format-iceberg/src/main/java/org/apache/amoro/mixed/BasicMixedIcebergCatalog.java

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ public class BasicMixedIcebergCatalog implements MixedFormatCatalog {
6565
privateAmsClientclient;
6666
privateMixedTablestables;
6767
privateSupportsNamespacesasNamespaceCatalog;
68+
privateStringseparator;
6869

6970
@Override
7071
publicStringname() {
@@ -93,6 +94,7 @@ public void initialize(String name, Map<String, String> properties, TableMetaSto
9394
}
9495
this.databaseFilterPattern =databaseFilterPattern;
9596
this.catalogProperties =properties;
97+
this.separator =tableStoreSeparator();
9698
this.tables =newMixedTables(metaStore,properties,icebergCatalog());
9799
if (properties.containsKey(CatalogMetaProperties.AMS_URI)) {
98100
this.client =newPooledAmsClient(properties.get(CatalogMetaProperties.AMS_URI));
@@ -221,7 +223,13 @@ protected MixedTables newMixedTables(
221223
TableMetaStoremetaStore,
222224
Map<String,String>catalogProperties,
223225
org.apache.iceberg.catalog.CatalogicebergCatalog) {
224-
returnnewMixedTables(metaStore,catalogProperties,icebergCatalog);
226+
returnnewMixedTables(metaStore,catalogProperties,icebergCatalog,separator);
227+
}
228+
229+
protectedStringtableStoreSeparator() {
230+
returncatalogProperties.getOrDefault(
231+
CatalogMetaProperties.MIXED_FORMAT_TABLE_STORE_SEPARATOR,
232+
CatalogMetaProperties.MIXED_FORMAT_TABLE_STORE_SEPARATOR_DEFAULT);
225233
}
226234

227235
privateorg.apache.iceberg.catalog.TableIdentifiertoIcebergTableIdentifier(

‎amoro-format-iceberg/src/main/java/org/apache/amoro/mixed/InternalMixedIcebergCatalog.java

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
importorg.apache.amoro.TableFormat;
2222
importorg.apache.amoro.shade.guava32.com.google.common.base.Preconditions;
2323
importorg.apache.amoro.shade.guava32.com.google.common.collect.Maps;
24+
importorg.apache.amoro.table.MixedTable;
2425
importorg.apache.amoro.table.PrimaryKeySpec;
2526
importorg.apache.amoro.table.TableMetaStore;
2627
importorg.apache.hadoop.conf.Configuration;
@@ -43,7 +44,6 @@
4344
publicclassInternalMixedIcebergCatalogextendsBasicMixedIcebergCatalog {
4445

4546
publicstaticfinalStringCHANGE_STORE_SEPARATOR ="@";
46-
publicstaticfinalStringCHANGE_STORE_NAME ="change";
4747

4848
publicstaticfinalStringHTTP_HEADER_LIST_TABLE_FILTER ="LIST-TABLE-FILTER";
4949

@@ -81,29 +81,23 @@ public List<org.apache.amoro.table.TableIdentifier> listTables(String database)
8181
@Override
8282
protectedMixedTablesnewMixedTables(
8383
TableMetaStoremetaStore,Map<String,String>catalogProperties,CatalogicebergCatalog) {
84-
returnnewInternalMixedTables(metaStore,catalogProperties,icebergCatalog);
84+
returnnewInternalMixedTables(
85+
metaStore,catalogProperties,icebergCatalog,tableStoreSeparator());
86+
}
87+
88+
@Override
89+
protectedStringtableStoreSeparator() {
90+
returnCHANGE_STORE_SEPARATOR;
8591
}
8692

8793
staticclassInternalMixedTablesextendsMixedTables {
8894

8995
publicInternalMixedTables(
90-
TableMetaStoretableMetaStore,Map<String,String>catalogProperties,Catalogcatalog) {
91-
super(tableMetaStore,catalogProperties,catalog);
92-
}
93-
94-
/**
95-
* For internal table, using {table-name}@change as change store identifier, this identifier
96-
* cloud be recognized by AMS. Due to '@' is an invalid character of table name, the change
97-
* store identifier will never be conflict with other table name.
98-
*
99-
* @param baseIdentifier base store table identifier.
100-
* @return change store iceberg table identifier.
101-
*/
102-
@Override
103-
protectedTableIdentifiergenerateChangeStoreIdentifier(TableIdentifierbaseIdentifier) {
104-
returnTableIdentifier.of(
105-
baseIdentifier.namespace(),
106-
baseIdentifier.name() +CHANGE_STORE_SEPARATOR +CHANGE_STORE_NAME);
96+
TableMetaStoretableMetaStore,
97+
Map<String,String>catalogProperties,
98+
Catalogcatalog,
99+
Stringseparator) {
100+
super(tableMetaStore,catalogProperties,catalog,separator);
107101
}
108102

109103
/**
@@ -121,6 +115,13 @@ protected Table createChangeStore(
121115
returntableMetaStore.doAs(() ->icebergCatalog.loadTable(changeIdentifier));
122116
}
123117

118+
@Override
119+
protectedTableIdentifiergenerateChangeStoreIdentifier(TableIdentifierbaseIdentifier) {
120+
returnTableIdentifier.of(
121+
baseIdentifier.namespace(),
122+
baseIdentifier.name() +CHANGE_STORE_SEPARATOR +MixedTable.CHANGE_STORE_IDENTIFIER);
123+
}
124+
124125
/**
125126
* The change store will be dropped automatically by AMS when dropping the base store, so we do
126127
* nothing here

‎amoro-format-iceberg/src/main/java/org/apache/amoro/mixed/MixedTables.java

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
importorg.apache.amoro.TableFormat;
2222
importorg.apache.amoro.io.AuthenticatedFileIO;
2323
importorg.apache.amoro.io.AuthenticatedFileIOs;
24-
importorg.apache.amoro.properties.CatalogMetaProperties;
2524
importorg.apache.amoro.shade.guava32.com.google.common.collect.Maps;
2625
importorg.apache.amoro.table.BaseTable;
2726
importorg.apache.amoro.table.BasicKeyedTable;
@@ -32,6 +31,7 @@
3231
importorg.apache.amoro.table.TableMetaStore;
3332
importorg.apache.amoro.table.UnkeyedTable;
3433
importorg.apache.amoro.utils.MixedFormatCatalogUtil;
34+
importorg.apache.amoro.utils.MixedTableUtil;
3535
importorg.apache.amoro.utils.TablePropertyUtil;
3636
importorg.apache.iceberg.PartitionSpec;
3737
importorg.apache.iceberg.Schema;
@@ -47,16 +47,20 @@
4747
publicclassMixedTables {
4848
privatestaticfinalLoggerLOG =LoggerFactory.getLogger(MixedTables.class);
4949

50-
protectedTableMetaStoretableMetaStore;
51-
protectedCatalogicebergCatalog;
52-
53-
protectedMap<String,String>catalogProperties;
50+
protectedfinalTableMetaStoretableMetaStore;
51+
protectedfinalCatalogicebergCatalog;
52+
protectedfinalMap<String,String>catalogProperties;
53+
protectedfinalStringseparator;
5454

5555
publicMixedTables(
56-
TableMetaStoretableMetaStore,Map<String,String>catalogProperties,Catalogcatalog) {
56+
TableMetaStoretableMetaStore,
57+
Map<String,String>catalogProperties,
58+
Catalogcatalog,
59+
Stringseparator) {
5760
this.tableMetaStore =tableMetaStore;
5861
this.icebergCatalog =catalog;
5962
this.catalogProperties =catalogProperties;
63+
this.separator =separator;
6064
}
6165

6266
/**
@@ -86,12 +90,9 @@ public TableIdentifier parseChangeIdentifier(Table base) {
8690
* @return change store table identifier.
8791
*/
8892
protectedTableIdentifiergenerateChangeStoreIdentifier(TableIdentifierbaseIdentifier) {
89-
Stringseparator =
90-
catalogProperties.getOrDefault(
91-
CatalogMetaProperties.MIXED_FORMAT_TABLE_STORE_SEPARATOR,
92-
CatalogMetaProperties.MIXED_FORMAT_TABLE_STORE_SEPARATOR_DEFAULT);
9393
returnTableIdentifier.of(
94-
baseIdentifier.namespace(),baseIdentifier.name() +separator +"change" +separator);
94+
baseIdentifier.namespace(),
95+
MixedTableUtil.changeStoreName(baseIdentifier.name(),separator));
9596
}
9697

9798
/**

‎amoro-format-iceberg/src/main/java/org/apache/amoro/table/MixedTable.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@
3131
/** Represents an mixed-format table. */
3232
publicinterfaceMixedTableextendsSerializable {
3333

34+
StringCHANGE_STORE_IDENTIFIER ="change";
35+
3436
/** Returns the {@link TableIdentifier} of this table */
3537
TableIdentifierid();
3638

0 commit comments

Comments
 (0)

[8]ページ先頭

©2009-2025 Movatter.jp