hive2.3.4
presto0.215
使用hive2.3.4的beeline连接presto报错
$ beeline -d com.facebook.presto.jdbc.PrestoDriver -u "jdbc:presto://localhost:8080/hive"
Error: Unrecognized connection property 'url' (state=,code=0)
增加--verbose查看详细报错
$ beeline -d com.facebook.presto.jdbc.PrestoDriver -u "jdbc:presto://localhost:8080/hive" --verbose
Error: Unrecognized connection property 'url' (state=,code=0)
java.sql.SQLException: Unrecognized connection property 'url'
at com.facebook.presto.jdbc.PrestoDriverUri.validateConnectionProperties(PrestoDriverUri.java:316)
at com.facebook.presto.jdbc.PrestoDriverUri.<init>(PrestoDriverUri.java:95)
at com.facebook.presto.jdbc.PrestoDriverUri.<init>(PrestoDriverUri.java:85)
at com.facebook.presto.jdbc.PrestoDriver.connect(PrestoDriver.java:87)
at java.sql.DriverManager.getConnection(DriverManager.java:664)
at java.sql.DriverManager.getConnection(DriverManager.java:208)
at org.apache.hive.beeline.DatabaseConnection.connect(DatabaseConnection.java:145)
at org.apache.hive.beeline.DatabaseConnection.getConnection(DatabaseConnection.java:209)
at org.apache.hive.beeline.Commands.connect(Commands.java:1641)
at org.apache.hive.beeline.Commands.connect(Commands.java:1536)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.hive.beeline.ReflectiveCommandHandler.execute(ReflectiveCommandHandler.java:56)
at org.apache.hive.beeline.BeeLine.execCommandWithPrefix(BeeLine.java:1273)
at org.apache.hive.beeline.BeeLine.dispatch(BeeLine.java:1312)
at org.apache.hive.beeline.BeeLine.connectUsingArgs(BeeLine.java:867)
at org.apache.hive.beeline.BeeLine.initArgs(BeeLine.java:776)
at org.apache.hive.beeline.BeeLine.begin(BeeLine.java:1010)
at org.apache.hive.beeline.BeeLine.mainWithInputRedirection(BeeLine.java:519)
at org.apache.hive.beeline.BeeLine.main(BeeLine.java:501)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.hadoop.util.RunJar.run(RunJar.java:221)
at org.apache.hadoop.util.RunJar.main(RunJar.java:136)
这里是connect过程报错，connect参数为
Usage: connect <url> <username> <password> [driver]
跟进hive和presto代码
org.apache.hive.beeline.BeeLine
boolean dispatch(String line) {
...
if (isBeeLine) {
if (line.startsWith(COMMAND_PREFIX)) {
// handle SQLLine command in beeline which starts with ! and does not end with ;
return execCommandWithPrefix(line);
} else {
return commands.sql(line, getOpts().getEntireLineAsCommand());
}
} else {
return commands.sql(line, getOpts().getEntireLineAsCommand());
}
beeline中的命令和sql都会经过dispatch，命令执行的是execCommandWithPrefix，然后会反射调用Commands
org.apache.hive.beeline.Commands
public boolean connect(String line) throws Exception {
String example = "Usage: connect <url> <username> <password> [driver]"
+ BeeLine.getSeparator();
String[] parts = beeLine.split(line);
if (parts == null) {
return false;
}
if (parts.length < 2) {
return beeLine.error(example);
}
String url = parts.length < 2 ? null : parts[1];
String user = parts.length < 3 ? null : parts[2];
String pass = parts.length < 4 ? null : parts[3];
String driver = parts.length < 5 ? null : parts[4];
Properties props = new Properties();
if (url != null) {
String saveUrl = getUrlToUse(url);
props.setProperty(JdbcConnectionParams.PROPERTY_URL, saveUrl);
}
String value = null;
if (driver != null) {
props.setProperty(JdbcConnectionParams.PROPERTY_DRIVER, driver);
} else {
value = Utils.parsePropertyFromUrl(url, JdbcConnectionParams.PROPERTY_DRIVER);
if (value != null) {
props.setProperty(JdbcConnectionParams.PROPERTY_DRIVER, value);
}
}
if (user != null) {
props.setProperty(JdbcConnectionParams.AUTH_USER, user);
} else {
value = Utils.parsePropertyFromUrl(url, JdbcConnectionParams.AUTH_USER);
if (value != null) {
props.setProperty(JdbcConnectionParams.AUTH_USER, value);
}
}
if (pass != null) {
props.setProperty(JdbcConnectionParams.AUTH_PASSWD, pass);
} else {
value = Utils.parsePropertyFromUrl(url, JdbcConnectionParams.AUTH_PASSWD);
if (value != null) {
props.setProperty(JdbcConnectionParams.AUTH_PASSWD, value);
}
}
value = Utils.parsePropertyFromUrl(url, JdbcConnectionParams.AUTH_TYPE);
if (value != null) {
props.setProperty(JdbcConnectionParams.AUTH_TYPE, value);
}
return connect(props);
}
public boolean connect(Properties props) throws IOException {
String url = getProperty(props, new String[] {
JdbcConnectionParams.PROPERTY_URL,
"javax.jdo.option.ConnectionURL",
"ConnectionURL",
});
String driver = getProperty(props, new String[] {
JdbcConnectionParams.PROPERTY_DRIVER,
"javax.jdo.option.ConnectionDriverName",
"ConnectionDriverName",
});
String username = getProperty(props, new String[] {
JdbcConnectionParams.AUTH_USER,
"javax.jdo.option.ConnectionUserName",
"ConnectionUserName",
});
String password = getProperty(props, new String[] {
JdbcConnectionParams.AUTH_PASSWD,
"javax.jdo.option.ConnectionPassword",
"ConnectionPassword",
});
if (url == null || url.length() == 0) {
return beeLine.error("Property \"url\" is required");
}
if (driver == null || driver.length() == 0) {
if (!beeLine.scanForDriver(url)) {
return beeLine.error(beeLine.loc("no-driver", url));
}
}
String auth = getProperty(props, new String[] {JdbcConnectionParams.AUTH_TYPE});
if (auth == null) {
auth = beeLine.getOpts().getAuthType();
if (auth != null) {
props.setProperty(JdbcConnectionParams.AUTH_TYPE, auth);
}
}
beeLine.info("Connecting to " + url);
if (Utils.parsePropertyFromUrl(url, JdbcConnectionParams.AUTH_PRINCIPAL) == null) {
String urlForPrompt = url.substring(0, url.contains(";") ? url.indexOf(';') : url.length());
if (username == null) {
username = beeLine.getConsoleReader().readLine("Enter username for " + urlForPrompt + ": ");
}
props.setProperty(JdbcConnectionParams.AUTH_USER, username);
if (password == null) {
password = beeLine.getConsoleReader().readLine("Enter password for " + urlForPrompt + ": ",
new Character('*'));
}
props.setProperty(JdbcConnectionParams.AUTH_PASSWD, password);
}
try {
beeLine.getDatabaseConnections().setConnection(
new DatabaseConnection(beeLine, driver, url, props));
beeLine.getDatabaseConnection().getConnection();
if (!beeLine.isBeeLine()) {
beeLine.updateOptsForCli();
}
beeLine.runInit();
beeLine.setCompletions();
beeLine.getOpts().setLastConnectedUrl(url);
return true;
} catch (SQLException sqle) {
beeLine.getDatabaseConnections().remove();
return beeLine.error(sqle);
} catch (IOException ioe) {
return beeLine.error(ioe);
}
}
注意props字段的赋值过程
org.apache.hive.beeline.DatabaseConnection
public DatabaseConnection(BeeLine beeLine, String driver, String url,
Properties info) throws SQLException {
this.beeLine = beeLine;
this.driver = driver;
this.url = url;
this.info = info;
} boolean connect() throws SQLException {
...
Map<String, String> hiveVars = beeLine.getOpts().getHiveVariables();
if (hiveVars != null){
for (Map.Entry<String, String> var : hiveVars.entrySet()) {
info.put(HIVE_VAR_PREFIX + var.getKey(), var.getValue());
}
} Map<String, String> hiveConfVars = beeLine.getOpts().getHiveConfVariables();
if (hiveConfVars != null){
for (Map.Entry<String, String> var : hiveConfVars.entrySet()) {
info.put(HIVE_CONF_PREFIX + var.getKey(), var.getValue());
}
} if (isDriverRegistered) {
// if the driver registered in the driver manager, get the connection via the driver manager
setConnection(DriverManager.getConnection(getUrl(), info));
} else {
Commands中props赋值给DatabaseConnection的info
java.sql.DriverManager
public static Connection getConnection(String url,
java.util.Properties info) throws SQLException { return (getConnection(url, info, Reflection.getCallerClass()));
} private static Connection getConnection(
String url, java.util.Properties info, Class<?> caller) throws SQLException {
...
Connection con = aDriver.driver.connect(url, info);
DatabaseConnection的info传递给DriverManager然后传递给Driver，下面看具体的PrestoDriver
com.facebook.presto.jdbc.PrestoDriver
public Connection connect(String url, Properties info)
throws SQLException
{
if (!acceptsURL(url)) {
return null;
} PrestoDriverUri uri = new PrestoDriverUri(url, info); OkHttpClient.Builder builder = httpClient.newBuilder();
uri.setupClient(builder);
QueryExecutor executor = new QueryExecutor(builder.build()); return new PrestoConnection(uri, executor);
}
info传递给PrestoDriverUri进行初始化
com.facebook.presto.jdbc.PrestoDriverUri
public PrestoDriverUri(String url, Properties driverProperties)
throws SQLException
{
this(parseDriverUrl(url), driverProperties);
} private PrestoDriverUri(URI uri, Properties driverProperties)
throws SQLException
{
this.uri = requireNonNull(uri, "uri is null");
address = HostAndPort.fromParts(uri.getHost(), uri.getPort());
properties = mergeConnectionProperties(uri, driverProperties); validateConnectionProperties(properties); // enable SSL by default for standard port
useSecureConnection = SSL.getValue(properties).orElse(uri.getPort() == 443); initCatalogAndSchema();
} private static void validateConnectionProperties(Properties connectionProperties)
throws SQLException
{
for (String propertyName : connectionProperties.stringPropertyNames()) {
if (ConnectionProperties.forKey(propertyName) == null) {
throw new SQLException(format("Unrecognized connection property '%s'", propertyName));
}
} for (ConnectionProperty<?> property : ConnectionProperties.allProperties()) {
property.validate(connectionProperties);
}
}
所以报错“Unrecognized connection property 'url' ”意味着ConnectionProperties.forKey("url") == null
com.facebook.presto.jdbc.ConnectionProperties
ConnectionProperties中确实没有url，所以presto与hive的beeline不兼容；
尝试修改PrestoDriverUri.java，去掉校验：
com.facebook.presto.jdbc.PrestoDriverUri
private static void validateConnectionProperties(Properties connectionProperties)
throws SQLException
{
// for (String propertyName : connectionProperties.stringPropertyNames()) {
// if (ConnectionProperties.forKey(propertyName) == null) {
// throw new SQLException(format("Unrecognized connection property '%s'", propertyName));
// }
// } for (ConnectionProperty<?> property : ConnectionProperties.allProperties()) {
property.validate(connectionProperties);
}
}
将以上几行注释
简单修改jar包中的class的方法，参考：https://www.cnblogs.com/barneywill/p/10552322.html
编译的时候发现一直报错，
presto-jdbc/com/facebook/presto/jdbc/PrestoDriverUri.java:145: error: method setupSocksProxy in class OkHttpUtil cannot be applied to given types;
setupSocksProxy(builder, SOCKS_PROXY.getValue(properties));
^
required: Builder,Optional<com.google.common.net.HostAndPort>
found: Builder,Optional<com.facebook.presto.jdbc.internal.guava.net.HostAndPort>
reason: argument mismatch; Optional<com.facebook.presto.jdbc.internal.guava.net.HostAndPort> cannot be converted to Optional<com.google.common.net.HostAndPort>
presto-jdbc/com/facebook/presto/jdbc/PrestoDriverUri.java:146: error: method setupHttpProxy in class OkHttpUtil cannot be applied to given types;
setupHttpProxy(builder, HTTP_PROXY.getValue(properties));
^
required: Builder,Optional<com.google.common.net.HostAndPort>
found: Builder,Optional<com.facebook.presto.jdbc.internal.guava.net.HostAndPort>
reason: argument mismatch; Optional<com.facebook.presto.jdbc.internal.guava.net.HostAndPort> cannot be converted to Optional<com.google.common.net.HostAndPort>
2 errors
然后发现原来是presto-jdbc打包的时候设置了relocation，会将一些包做shade并且改package，详见：
https://github.com/prestodb/presto/blob/master/presto-jdbc/pom.xml
<properties>
<air.main.basedir>${project.parent.basedir}</air.main.basedir>
<shadeBase>com.facebook.presto.jdbc.internal</shadeBase>
</properties>
...
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<createSourcesJar>true</createSourcesJar>
<shadeSourcesContent>true</shadeSourcesContent>
<dependencyReducedPomLocation>${project.build.directory}/pom.xml</dependencyReducedPomLocation>
<relocations>
<relocation>
<pattern>com.facebook.presto.client</pattern>
<shadedPattern>${shadeBase}.client</shadedPattern>
</relocation>
<relocation>
<pattern>com.facebook.presto.spi</pattern>
<shadedPattern>${shadeBase}.spi</shadedPattern>
</relocation>
<relocation>
<pattern>com.fasterxml.jackson</pattern>
<shadedPattern>${shadeBase}.jackson</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.common</pattern>
<shadedPattern>${shadeBase}.guava</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.thirdparty</pattern>
<shadedPattern>${shadeBase}.guava</shadedPattern>
</relocation>
<relocation>
<pattern>io.airlift</pattern>
<shadedPattern>${shadeBase}.airlift</shadedPattern>
</relocation>
<relocation>
<pattern>javax.annotation</pattern>
<shadedPattern>${shadeBase}.javax.annotation</shadedPattern>
</relocation>
<relocation>
<pattern>javax.inject</pattern>
<shadedPattern>${shadeBase}.inject</shadedPattern>
</relocation>
<relocation>
<pattern>org.openjdk.jol</pattern>
<shadedPattern>${shadeBase}.jol</shadedPattern>
</relocation>
<relocation>
<pattern>org.joda.time</pattern>
<shadedPattern>${shadeBase}.joda.time</shadedPattern>
</relocation>
<relocation>
<pattern>okhttp3</pattern>
<shadedPattern>${shadeBase}.okhttp3</shadedPattern>
</relocation>
<relocation>
<pattern>okio</pattern>
<shadedPattern>${shadeBase}.okio</shadedPattern>
</relocation>
</relocations>
所以需要手工调整源代码中的package
okhttp3 -> com.facebook.presto.jdbc.internal.okhttp3
com.google.common -> com.facebook.presto.jdbc.internal.guava
com.facebook.presto.client -> com.facebook.presto.jdbc.internal.client
源代码地址：
https://github.com/prestodb/presto/blob/master/presto-jdbc/src/main/java/com/facebook/presto/jdbc/PrestoDriverUri.java
然后再编译就ok了，编译命令：
# javac -cp presto-jdbc-0.215.jar -d presto-jdbc presto-jdbc/com/facebook/presto/jdbc/PrestoDriverUri.java
然后重新打包，再使用beeline连接就ok了