feat: add snowflake plugin

chengluo · chengluo · commit 33fb659c1020 · 2023-02-21T19:03:35.000+08:00
diff --git a/server/api-service/README.md b/server/api-service/README.md
@@ -62,7 +62,7 @@ IDEA 2021.3.2 (Community Edition):
     </tr>
     <tr>
         <td>VM options </td>
-        <td>-Dpf4j.mode=development -Dpf4j.pluginsDir=openblocks-plugins -Dspring.profiles.active=openblocks -XX:+AllowRedefinitionToAddDeleteMethods </td>
+        <td>-Dpf4j.mode=development -Dpf4j.pluginsDir=openblocks-plugins -Dspring.profiles.active=openblocks -XX:+AllowRedefinitionToAddDeleteMethods --add-opens java.base/java.nio=ALL-UNNAMED</td>
     </tr>
     <tr>
         <td>Main class </td>
diff --git a/server/api-service/openblocks-domain/src/main/java/com/openblocks/domain/datasource/service/impl/DatasourceStructureServiceImpl.java b/server/api-service/openblocks-domain/src/main/java/com/openblocks/domain/datasource/service/impl/DatasourceStructureServiceImpl.java
@@ -99,7 +99,7 @@ private Mono<DatasourceStructure> getLatestStructure(Datasource datasource,
             QueryExecutor<? extends DatasourceConnectionConfig, Object, ? extends QueryExecutionContext> queryExecutor) {
 
         return connectionContextService.getOrCreateConnection(datasource)
-                .flatMap(connectionContext -> queryExecutor.doGetStructure(connectionContext.connection())
+                .flatMap(connectionContext -> queryExecutor.doGetStructure(connectionContext.connection(), datasource.getDetailConfig())
                         .timeout(Duration.ofMillis(GET_STRUCTURE_DEFAULT_TIMEOUT_MILLIS))
                         .doOnError(connectionContext::onQueryError)
                         .onErrorMap(TimeoutException.class, e -> new BizException(BizError.PLUGIN_EXECUTION_TIMEOUT, "PLUGIN_EXECUTION_TIMEOUT",
diff --git a/server/api-service/openblocks-domain/src/main/java/com/openblocks/domain/plugin/service/impl/DatasourceMetaInfoServiceImpl.java b/server/api-service/openblocks-domain/src/main/java/com/openblocks/domain/plugin/service/impl/DatasourceMetaInfoServiceImpl.java
@@ -117,6 +117,15 @@ public class DatasourceMetaInfoServiceImpl implements DatasourceMetaInfoService
             .connectionPool(ClientBasedConnectionPool.class)
             .build();
 
+
+    private static final DatasourceMetaInfo SNOWFLAKE = DatasourceMetaInfo.builder()
+            .type("snowflake")
+            .displayName("Snowflake")
+            .pluginExecutorKey("snowflake-plugin")
+            .hasStructureInfo(true)
+            .connectionPool(ClientBasedConnectionPool.class)
+            .build();
+
     private static final DatasourceMetaInfo GOOGLE_SHEETS = DatasourceMetaInfo.builder()
             .type("googleSheets")
             .displayName("Google Sheets")
diff --git a/server/api-service/openblocks-plugins/clickHousePlugin/src/main/java/com/openblocks/plugin/clickhouse/ClickHouseQueryExecutor.java b/server/api-service/openblocks-plugins/clickHousePlugin/src/main/java/com/openblocks/plugin/clickhouse/ClickHouseQueryExecutor.java
@@ -108,7 +108,8 @@ public Mono<QueryExecutionResult> executeQuery(HikariDataSource hikariDataSource
     }
 
     @Override
-    public Mono<DatasourceStructure> getStructure(HikariDataSource hikariDataSource) {
+    public Mono<DatasourceStructure> getStructure(HikariDataSource hikariDataSource,
+            ClickHouseDatasourceConfig connectionConfig) {
 
         return Mono.fromCallable(() -> {
                     Connection connection = getConnection(hikariDataSource);
diff --git a/server/api-service/openblocks-plugins/mongoPlugin/src/main/java/com/openblocks/plugin/mongo/MongoPlugin.java b/server/api-service/openblocks-plugins/mongoPlugin/src/main/java/com/openblocks/plugin/mongo/MongoPlugin.java
@@ -471,7 +471,8 @@ public Mono<DatasourceTestResult> testConnection(MongoDatasourceConfig connectio
         }
 
         @Override
-        public Mono<DatasourceStructure> getStructure(MongoConnection mongoClient) {
+        public Mono<DatasourceStructure> getStructure(MongoConnection mongoClient,
+                MongoDatasourceConfig connectionConfig) {
             final DatasourceStructure structure = new DatasourceStructure();
             List<DatasourceStructure.Table> tables = new ArrayList<>();
             structure.setTables(tables);
diff --git a/server/api-service/openblocks-plugins/mssqlPlugin/src/main/java/com/openblocks/plugin/mssql/MssqlQueryExecutor.java b/server/api-service/openblocks-plugins/mssqlPlugin/src/main/java/com/openblocks/plugin/mssql/MssqlQueryExecutor.java
@@ -119,7 +119,8 @@ public Mono<QueryExecutionResult> executeQuery(HikariDataSource hikariDataSource
     }
 
     @Override
-    public Mono<DatasourceStructure> getStructure(HikariDataSource hikariDataSource) {
+    public Mono<DatasourceStructure> getStructure(HikariDataSource hikariDataSource,
+            MssqlDatasourceConfig connectionConfig) {
         return Mono.fromCallable(() -> {
                     Connection connection = getConnection(hikariDataSource);
 
diff --git a/server/api-service/openblocks-plugins/mysqlPlugin/src/main/java/com/openblocks/plugin/mysql/MysqlQueryExecutor.java b/server/api-service/openblocks-plugins/mysqlPlugin/src/main/java/com/openblocks/plugin/mysql/MysqlQueryExecutor.java
@@ -22,6 +22,7 @@
 import com.openblocks.sdk.exception.PluginException;
 import com.openblocks.sdk.models.DatasourceStructure;
 import com.openblocks.sdk.models.DatasourceStructure.Table;
+import com.openblocks.sdk.plugin.common.sql.SqlBasedDatasourceConnectionConfig;
 import com.openblocks.sdk.plugin.sqlcommand.GuiSqlCommand;
 import com.openblocks.sdk.plugin.sqlcommand.command.mysql.MysqlBulkInsertCommand;
 import com.openblocks.sdk.plugin.sqlcommand.command.mysql.MysqlBulkUpdateCommand;
@@ -42,7 +43,8 @@ public MysqlQueryExecutor() {
 
     @Nonnull
     @Override
-    protected DatasourceStructure getDatabaseMetadata(Connection connection) {
+    protected DatasourceStructure getDatabaseMetadata(Connection connection,
+            SqlBasedDatasourceConnectionConfig connectionConfig) {
         Map<String, Table> tablesByName = new LinkedHashMap<>();
         try (Statement statement = connection.createStatement()) {
             parseTableAndColumns(tablesByName, statement);
diff --git a/server/api-service/openblocks-plugins/mysqlPlugin/src/main/java/com/openblocks/plugin/mysql/OldMysqlQueryExecutor.java b/server/api-service/openblocks-plugins/mysqlPlugin/src/main/java/com/openblocks/plugin/mysql/OldMysqlQueryExecutor.java
@@ -202,7 +202,8 @@ private HikariDataSource getHikariDataSource(HikariPerfWrapper hikariDataSource)
 
     @Nonnull
     @Override
-    public DatasourceStructure blockingGetStructure(HikariPerfWrapper hikariPerfWrapper) {
+    public DatasourceStructure blockingGetStructure(HikariPerfWrapper hikariPerfWrapper,
+            MysqlDatasourceConfig connectionConfig) {
 
         Map<String, Table> tablesByName = new LinkedHashMap<>();
         try (Connection connection = getConnection(getHikariDataSource(hikariPerfWrapper));
diff --git a/server/api-service/openblocks-plugins/oraclePlugin/src/main/java/com/openblocks/plugin/oracle/OraclePlugin.java b/server/api-service/openblocks-plugins/oraclePlugin/src/main/java/com/openblocks/plugin/oracle/OraclePlugin.java
@@ -119,7 +119,8 @@ public Mono<Void> destroyConnection(HikariDataSource hikariDataSource) {
         }
 
         @Override
-        public Mono<DatasourceStructure> getStructure(HikariDataSource hikariDataSource) {
+        public Mono<DatasourceStructure> getStructure(HikariDataSource hikariDataSource,
+                OracleDatasourceConfig connectionConfig) {
             return Mono.fromSupplier(() -> getStructure0(hikariDataSource))
                     .subscribeOn(querySharedScheduler());
         }
diff --git a/server/api-service/openblocks-plugins/pom.xml b/server/api-service/openblocks-plugins/pom.xml
@@ -94,5 +94,6 @@
         <module>oraclePlugin</module>
         <module>graphqlPlugin</module>
         <module>sqlBasedPlugin</module>
+        <module>snowflakePlugin</module>
     </modules>
 </project>
diff --git a/server/api-service/openblocks-plugins/postgresPlugin/src/main/java/com/openblocks/plugin/postgres/PostgresExecutor.java b/server/api-service/openblocks-plugins/postgresPlugin/src/main/java/com/openblocks/plugin/postgres/PostgresExecutor.java
@@ -349,7 +349,8 @@ private List<LocaleMessage> populateHintMessages(List<String> columnNames) {
 
     @Nonnull
     @Override
-    public DatasourceStructure blockingGetStructure(HikariDataSource hikariDataSource) {
+    public DatasourceStructure blockingGetStructure(HikariDataSource hikariDataSource,
+            PostgresDatasourceConfig connectionConfig) {
 
         DatasourceStructure structure = new DatasourceStructure();
         Map<String, DatasourceStructure.Table> tablesByName = new LinkedHashMap<>();
diff --git a/server/api-service/openblocks-plugins/snowflakePlugin/plugin.properties b/server/api-service/openblocks-plugins/snowflakePlugin/plugin.properties
@@ -0,0 +1,5 @@
+plugin.id=snowflake-plugin
+plugin.class=com.openblocks.plugin.snowflake.SnowflakePlugin
+plugin.version=1.0-SNAPSHOT
+plugin.provider=dev@openblocks.dev
+plugin.dependencies=
diff --git a/server/api-service/openblocks-plugins/snowflakePlugin/pom.xml b/server/api-service/openblocks-plugins/snowflakePlugin/pom.xml
@@ -0,0 +1,96 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.openblocks</groupId>
+        <artifactId>openblocks-plugins</artifactId>
+        <version>1.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>snowflakePlugin</artifactId>
+
+    <properties>
+        <maven.compiler.source>17</maven.compiler.source>
+        <maven.compiler.target>17</maven.compiler.target>
+        <java.version>17</java.version>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+
+        <plugin.id>snowflake-plugin</plugin.id>
+        <plugin.class>com.openblocks.plugin.snowflake.SnowflakePlugin</plugin.class>
+        <plugin.version>1.0-SNAPSHOT</plugin.version>
+        <plugin.provider>dev@openblocks.dev</plugin.provider>
+        <plugin.dependencies/>
+    </properties>
+
+    <dependencies>
+
+        <dependency>
+            <groupId>net.snowflake</groupId>
+            <artifactId>snowflake-jdbc</artifactId>
+            <version>3.13.27</version>
+        </dependency>
+        <dependency>
+            <groupId>com.openblocks</groupId>
+            <artifactId>sqlBasedPlugin</artifactId>
+            <version>1.0-SNAPSHOT</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>testcontainers</artifactId>
+            <version>1.16.3</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.2.4</version>
+                <executions>
+                    <execution>
+                        <id>shade-plugin-jar</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <minimizeJar>false</minimizeJar>
+                            <transformers>
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <manifestEntries>
+                                        <Plugin-Id>${plugin.id}</Plugin-Id>
+                                        <Plugin-Class>${plugin.class}</Plugin-Class>
+                                        <Plugin-Version>${plugin.version}</Plugin-Version>
+                                        <Plugin-Provider>${plugin.provider}</Plugin-Provider>
+                                    </manifestEntries>
+                                </transformer>
+                            </transformers>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <artifactId>maven-dependency-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>copy-dependencies</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>copy-dependencies</goal>
+                        </goals>
+                        <configuration>
+                            <includeScope>runtime</includeScope>
+                            <outputDirectory>${project.build.directory}/lib</outputDirectory>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>
diff --git a/server/api-service/openblocks-plugins/snowflakePlugin/src/main/java/com/openblocks/plugin/snowflake/SnowflakeConnector.java b/server/api-service/openblocks-plugins/snowflakePlugin/src/main/java/com/openblocks/plugin/snowflake/SnowflakeConnector.java
@@ -0,0 +1,42 @@
+package com.openblocks.plugin.snowflake;
+
+import org.apache.commons.lang3.StringUtils;
+import org.pf4j.Extension;
+
+import com.openblocks.plugin.sql.SqlBasedConnector;
+import com.zaxxer.hikari.HikariConfig;
+
+@Extension
+public class SnowflakeConnector extends SqlBasedConnector<SnowflakeDatasourceConfig> {
+
+    private static final String JDBC_DRIVER = "net.snowflake.client.jdbc.SnowflakeDriver";
+
+    public SnowflakeConnector() {
+        super(50);
+    }
+
+    @Override
+    protected String getJdbcDriver() {
+        return JDBC_DRIVER;
+    }
+
+    @Override
+    protected void setUpConfigs(SnowflakeDatasourceConfig datasourceConfig, HikariConfig config) {
+        String host = datasourceConfig.getHost();
+        String database = datasourceConfig.getDatabase();
+
+        String url = "jdbc:snowflake://" + host + ".snowflakecomputing.com/";
+        config.setJdbcUrl(url);
+        config.addDataSourceProperty("db", database);
+        config.addDataSourceProperty("user", datasourceConfig.getUsername());
+        config.addDataSourceProperty("password", datasourceConfig.getPassword());
+
+        datasourceConfig.getExtParams()
+                .forEach((key, value) -> {
+                    if (StringUtils.isBlank(key) || value == null || StringUtils.isBlank(String.valueOf(value))) {
+                        return;
+                    }
+                    config.addDataSourceProperty(key, value);
+                });
+    }
+}
diff --git a/server/api-service/openblocks-plugins/snowflakePlugin/src/main/java/com/openblocks/plugin/snowflake/SnowflakeDatasourceConfig.java b/server/api-service/openblocks-plugins/snowflakePlugin/src/main/java/com/openblocks/plugin/snowflake/SnowflakeDatasourceConfig.java
@@ -0,0 +1,27 @@
+package com.openblocks.plugin.snowflake;
+
+import java.util.Map;
+
+import com.openblocks.sdk.plugin.common.sql.SqlBasedDatasourceConnectionConfig;
+
+import lombok.Builder;
+import lombok.ToString;
+import lombok.extern.slf4j.Slf4j;
+
+@ToString
+@Slf4j
+public class SnowflakeDatasourceConfig extends SqlBasedDatasourceConnectionConfig {
+
+    @Builder
+    public SnowflakeDatasourceConfig(String database, String username, String password, String host,
+            Long port, boolean usingSsl, String serverTimezone,
+            boolean isReadonly, boolean enableTurnOffPreparedStatement, Map<String, Object> extParams) {
+        super(database, username, password, host, port, usingSsl, serverTimezone, isReadonly, enableTurnOffPreparedStatement, extParams);
+    }
+
+    @Override
+    protected long defaultPort() {
+        return -1;
+    }
+
+}
diff --git a/server/api-service/openblocks-plugins/snowflakePlugin/src/main/java/com/openblocks/plugin/snowflake/SnowflakePlugin.java b/server/api-service/openblocks-plugins/snowflakePlugin/src/main/java/com/openblocks/plugin/snowflake/SnowflakePlugin.java
@@ -0,0 +1,12 @@
+package com.openblocks.plugin.snowflake;
+
+import org.pf4j.Plugin;
+import org.pf4j.PluginWrapper;
+
+public class SnowflakePlugin extends Plugin {
+
+    public SnowflakePlugin(PluginWrapper wrapper) {
+        super(wrapper);
+    }
+
+}
diff --git a/server/api-service/openblocks-plugins/snowflakePlugin/src/main/java/com/openblocks/plugin/snowflake/SnowflakeQueryExecutor.java b/server/api-service/openblocks-plugins/snowflakePlugin/src/main/java/com/openblocks/plugin/snowflake/SnowflakeQueryExecutor.java
diff --git a/server/api-service/openblocks-plugins/sqlBasedPlugin/src/main/java/com/openblocks/plugin/sql/SqlBasedConnector.java b/server/api-service/openblocks-plugins/sqlBasedPlugin/src/main/java/com/openblocks/plugin/sql/SqlBasedConnector.java
diff --git a/server/api-service/openblocks-plugins/sqlBasedPlugin/src/main/java/com/openblocks/plugin/sql/SqlBasedQueryExecutor.java b/server/api-service/openblocks-plugins/sqlBasedPlugin/src/main/java/com/openblocks/plugin/sql/SqlBasedQueryExecutor.java
diff --git a/server/api-service/openblocks-sdk/src/main/java/com/openblocks/sdk/plugin/common/BlockingQueryExecutor.java b/server/api-service/openblocks-sdk/src/main/java/com/openblocks/sdk/plugin/common/BlockingQueryExecutor.java
diff --git a/server/api-service/openblocks-sdk/src/main/java/com/openblocks/sdk/plugin/common/QueryExecutor.java b/server/api-service/openblocks-sdk/src/main/java/com/openblocks/sdk/plugin/common/QueryExecutor.java
diff --git a/server/api-service/openblocks-sdk/src/main/java/com/openblocks/sdk/plugin/common/sql/SqlBasedQueryExecutionContext.java b/server/api-service/openblocks-sdk/src/main/java/com/openblocks/sdk/plugin/common/sql/SqlBasedQueryExecutionContext.java

Original file line number	Diff line number	Diff line change
`@@ -108,7 +108,8 @@ public Mono<QueryExecutionResult> executeQuery(HikariDataSource hikariDataSource`
`108`	`108`	`}`
`109`	`109`
`110`	`110`	`@Override`
`111`		`- public Mono<DatasourceStructure> getStructure(HikariDataSource hikariDataSource) {`
	`111`	`+ public Mono<DatasourceStructure> getStructure(HikariDataSource hikariDataSource,`
	`112`	`+ ClickHouseDatasourceConfig connectionConfig) {`
`112`	`113`
`113`	`114`	`return Mono.fromCallable(() -> {`
`114`	`115`	`Connection connection = getConnection(hikariDataSource);`
Original file line number	Diff line number	Diff line change
`@@ -471,7 +471,8 @@ public Mono<DatasourceTestResult> testConnection(MongoDatasourceConfig connectio`
`471`	`471`	`}`
`472`	`472`
`473`	`473`	`@Override`
`474`		`- public Mono<DatasourceStructure> getStructure(MongoConnection mongoClient) {`
	`474`	`+ public Mono<DatasourceStructure> getStructure(MongoConnection mongoClient,`
	`475`	`+ MongoDatasourceConfig connectionConfig) {`
`475`	`476`	`final DatasourceStructure structure = new DatasourceStructure();`
`476`	`477`	`List<DatasourceStructure.Table> tables = new ArrayList<>();`
`477`	`478`	`structure.setTables(tables);`
Original file line number	Diff line number	Diff line change
`@@ -119,7 +119,8 @@ public Mono<QueryExecutionResult> executeQuery(HikariDataSource hikariDataSource`
`119`	`119`	`}`
`120`	`120`
`121`	`121`	`@Override`
`122`		`- public Mono<DatasourceStructure> getStructure(HikariDataSource hikariDataSource) {`
	`122`	`+ public Mono<DatasourceStructure> getStructure(HikariDataSource hikariDataSource,`
	`123`	`+ MssqlDatasourceConfig connectionConfig) {`
`123`	`124`	`return Mono.fromCallable(() -> {`
`124`	`125`	`Connection connection = getConnection(hikariDataSource);`
`125`	`126`
Original file line number	Diff line number	Diff line change
`@@ -119,7 +119,8 @@ public Mono<Void> destroyConnection(HikariDataSource hikariDataSource) {`
`119`	`119`	`}`
`120`	`120`
`121`	`121`	`@Override`
`122`		`- public Mono<DatasourceStructure> getStructure(HikariDataSource hikariDataSource) {`
	`122`	`+ public Mono<DatasourceStructure> getStructure(HikariDataSource hikariDataSource,`
	`123`	`+ OracleDatasourceConfig connectionConfig) {`
`123`	`124`	`return Mono.fromSupplier(() -> getStructure0(hikariDataSource))`
`124`	`125`	`.subscribeOn(querySharedScheduler());`
`125`	`126`	`}`