diff --git a/pom.xml b/pom.xml index ba08baeea5..3f2acc5db2 100644 --- a/pom.xml +++ b/pom.xml @@ -81,11 +81,12 @@ 2.3.0 1.15.0 32.0.1-jre - 3.4.2 - 1.19.4 + 3.5.0 + 2.46 1.5.4 3.0.0 4.13.2 + 5.9.3 1.8 1.0.0 3.1.0 @@ -328,14 +329,6 @@ commons-logging commons-logging-api - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-server - org.eclipse.jdt core @@ -413,14 +406,6 @@ commons-logging commons-logging-api - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-server - io.netty * @@ -773,6 +758,18 @@ ${junit.version} test + + org.junit.jupiter + junit-jupiter-api + ${junit.jupiter.version} + test + + + org.junit.vintage + junit-vintage-engine + ${junit.jupiter.version} + test + com.google.protobuf protobuf-java @@ -789,13 +786,13 @@ ${jettison.version} - com.sun.jersey + org.glassfish.jersey.core jersey-client ${jersey.version} - com.sun.jersey - jersey-json + org.glassfish.jersey.media + jersey-media-json-jackson ${jersey.version} @@ -822,6 +819,19 @@ + + + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.vintage + junit-vintage-engine + test + + + hadoop-shim tez-api diff --git a/tez-api/pom.xml b/tez-api/pom.xml index 58da12ae21..187e87d99b 100644 --- a/tez-api/pom.xml +++ b/tez-api/pom.xml @@ -94,12 +94,12 @@ junit - com.sun.jersey + org.glassfish.jersey.core jersey-client - com.sun.jersey - jersey-json + org.glassfish.jersey.media + jersey-media-json-jackson org.apache.hadoop diff --git a/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java b/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java index 501cc62ba8..54f50ac5f6 100644 --- a/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java +++ b/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java @@ -27,15 +27,16 @@ public class ATSConstants { /* Top level keys */ public static final String ENTITIES = "entities"; - public static final String ENTITY = "entity"; - public static final String ENTITY_TYPE = "entitytype"; + public static final String ENTITY_ID = "entityId"; + public static final String ENTITY_TYPE = "entityType"; public static final String EVENTS = "events"; - public static final String EVENT_TYPE = "eventtype"; + public static final String EVENT_TYPE = "eventType"; public static final String TIMESTAMP = "ts"; - public static final String EVENT_INFO = "eventinfo"; + public static final String EVENT_INFO = "eventInfo"; public static final String RELATED_ENTITIES = "relatedEntities"; - public static final String PRIMARY_FILTERS = "primaryfilters"; - public static final String OTHER_INFO = "otherinfo"; + public static final String PRIMARY_FILTERS = "primaryFilters"; + public static final String OTHER_INFO = "otherInfo"; + public static final String DOMAIN_ID = "domainId"; /* Section for related entities */ public static final String APPLICATION_ID = "applicationId"; diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java index 9e0a8d0dff..290490c033 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java @@ -29,6 +29,8 @@ import java.util.Set; import javax.annotation.Nullable; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -52,11 +54,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientHandlerException; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.UniformInterfaceException; -import com.sun.jersey.api.client.WebResource; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; @@ -212,7 +209,7 @@ public DAGStatus waitForCompletionWithStatusUpdates( @Override public void close() throws IOException { if (httpClient != null) { - httpClient.destroy(); + httpClient.close(); httpClient = null; } if (timelineReaderStrategy != null) { @@ -426,26 +423,14 @@ protected Map parseTaskStatsForVertexes() @VisibleForTesting protected JSONObject getJsonRootEntity(String url) throws TezException { try { - WebResource wr = getCachedHttpClient().resource(url); - ClientResponse response = wr.accept(MediaType.APPLICATION_JSON_TYPE) - .type(MediaType.APPLICATION_JSON_TYPE) - .get(ClientResponse.class); - - final ClientResponse.Status clientResponseStatus = response.getClientResponseStatus(); - if (clientResponseStatus != ClientResponse.Status.OK) { - throw new TezException("Failed to get response from YARN Timeline:" + - " errorCode:" + clientResponseStatus + ", url:" + url); - } - - return response.getEntity(JSONObject.class); - } catch (ClientHandlerException e) { + Client client = getCachedHttpClient(); + WebTarget target = client.target(url); + String json = target.request(MediaType.APPLICATION_JSON_TYPE) + .accept(MediaType.APPLICATION_JSON_TYPE) + .get(String.class); + return new JSONObject(json); + } catch (Exception e) { throw new TezException("Error processing response from YARN Timeline", e); - } catch (UniformInterfaceException e) { - throw new TezException("Error accessing content from YARN Timeline - unexpected response", e); - } catch (IllegalArgumentException e) { - throw new TezException("Error accessing content from YARN Timeline - invalid url", e); - } catch (IOException e) { - throw new TezException("Error failed to get http client", e); } } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/TimelineReaderFactory.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/TimelineReaderFactory.java index 14239697c9..d34a597eea 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/TimelineReaderFactory.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/TimelineReaderFactory.java @@ -30,11 +30,12 @@ import javax.net.ssl.HostnameVerifier; import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLSocketFactory; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authentication.client.AuthenticatedURL; import org.apache.hadoop.security.authentication.client.Authenticator; import org.apache.hadoop.security.authentication.client.ConnectionConfigurator; import org.apache.hadoop.security.ssl.SSLFactory; @@ -42,13 +43,10 @@ import org.apache.tez.dag.api.TezException; import com.google.common.annotations.VisibleForTesting; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.config.ClientConfig; -import com.sun.jersey.api.client.config.DefaultClientConfig; -import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory; -import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; -import com.sun.jersey.json.impl.provider.entity.JSONRootElementProvider; +import org.glassfish.jersey.client.ClientConfig; +import org.glassfish.jersey.client.HttpUrlConnectorProvider; +import org.glassfish.jersey.jackson.JacksonFeature; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -164,7 +162,7 @@ public Client getHttpClient() throws IOException { UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); UserGroupInformation realUgi = ugi.getRealUser(); String doAsUser; - ClientConfig clientConfig = new DefaultClientConfig(JSONRootElementProvider.App.class); + ClientConfig clientConfig = new ClientConfig().register(JacksonFeature.class); ConnectionConfigurator connectionConfigurator = getNewConnectionConf(useHttps, connTimeout, sslFactory); @@ -181,14 +179,16 @@ public Client getHttpClient() throws IOException { doAsUser = null; } - HttpURLConnectionFactory connectionFactory; + HttpUrlConnectorProvider.ConnectionFactory connectionFactory; try { connectionFactory = new TokenAuthenticatedURLConnectionFactory(connectionConfigurator, authenticator, doAsUser); } catch (TezException e) { throw new IOException("Fail to create TokenAuthenticatedURLConnectionFactory", e); } - return new Client(new URLConnectionClientHandler(connectionFactory), clientConfig); + + clientConfig.connectorProvider(new HttpUrlConnectorProvider().connectionFactory(connectionFactory)); + return ClientBuilder.newClient(clientConfig); } private static Authenticator getTokenAuthenticator() throws TezException { @@ -203,12 +203,12 @@ private static Authenticator getTokenAuthenticator() throws TezException { return ReflectionUtils.createClazzInstance(authenticatorClazzName); } - private static class TokenAuthenticatedURLConnectionFactory implements HttpURLConnectionFactory { + private static class TokenAuthenticatedURLConnectionFactory implements HttpUrlConnectorProvider.ConnectionFactory { private final Authenticator authenticator; private final ConnectionConfigurator connConfigurator; private final String doAsUser; - private final AuthenticatedURL.Token token; + private final Object token; public TokenAuthenticatedURLConnectionFactory(ConnectionConfigurator connConfigurator, Authenticator authenticator, @@ -221,9 +221,9 @@ public TokenAuthenticatedURLConnectionFactory(ConnectionConfigurator connConfigu } @Override - public HttpURLConnection getHttpURLConnection(URL url) throws IOException { + public HttpURLConnection getConnection(URL url) throws IOException { try { - AuthenticatedURL authenticatedURL= ReflectionUtils.createClazzInstance( + Object authenticatedURL = ReflectionUtils.createClazzInstance( DELEGATION_TOKEN_AUTHENTICATED_URL_CLAZZ_NAME, new Class[] { delegationTokenAuthenticatorClazz, ConnectionConfigurator.class @@ -231,7 +231,7 @@ public HttpURLConnection getHttpURLConnection(URL url) throws IOException { authenticator, connConfigurator }); - return ReflectionUtils.invokeMethod(authenticatedURL, + return (HttpURLConnection) ReflectionUtils.invokeMethod(authenticatedURL, delegationTokenAuthenticateURLOpenConnectionMethod, url, token, doAsUser); } catch (Exception e) { throw new IOException(e); @@ -265,13 +265,14 @@ public TimelineReaderPseudoAuthenticatedStrategy(final Configuration conf, @Override public Client getHttpClient() { - ClientConfig config = new DefaultClientConfig(JSONRootElementProvider.App.class); - HttpURLConnectionFactory urlFactory = new PseudoAuthenticatedURLConnectionFactory(connectionConf); - return new Client(new URLConnectionClientHandler(urlFactory), config); + ClientConfig config = new ClientConfig().register(JacksonFeature.class); + HttpUrlConnectorProvider.ConnectionFactory urlFactory = new PseudoAuthenticatedURLConnectionFactory(connectionConf); + config.connectorProvider(new HttpUrlConnectorProvider().connectionFactory(urlFactory)); + return ClientBuilder.newClient(config); } @VisibleForTesting - protected static class PseudoAuthenticatedURLConnectionFactory implements HttpURLConnectionFactory { + protected static class PseudoAuthenticatedURLConnectionFactory implements HttpUrlConnectorProvider.ConnectionFactory { private final ConnectionConfigurator connectionConf; public PseudoAuthenticatedURLConnectionFactory(ConnectionConfigurator connectionConf) { @@ -279,7 +280,7 @@ public PseudoAuthenticatedURLConnectionFactory(ConnectionConfigurator connection } @Override - public HttpURLConnection getHttpURLConnection(URL url) throws IOException { + public HttpURLConnection getConnection(URL url) throws IOException { String tokenString = (url.getQuery() == null ? "?" : "&") + "user.name=" + URLEncoder.encode(UserGroupInformation.getCurrentUser().getShortUserName(), "UTF8"); diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/client/TestATSHttpClient.java b/tez-api/src/test/java/org/apache/tez/dag/api/client/TestATSHttpClient.java index a4cab8f1fc..175c136a14 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/client/TestATSHttpClient.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/client/TestATSHttpClient.java @@ -95,7 +95,7 @@ public void testGetDagStatusSimple() throws TezException, JSONException, IOExcep final String jsonDagData = "{ " + - " otherinfo: { " + + " otherInfo: { " + " status: 'SUCCEEDED'," + " diagnostics: 'SAMPLE_DIAGNOSTICS'," + " counters: { counterGroups: [ " + @@ -108,9 +108,9 @@ public void testGetDagStatusSimple() throws TezException, JSONException, IOExcep "}"; final String jsonVertexData = "{entities:[ " + - "{otherinfo: {vertexName:'v1', numTasks:5,numFailedTasks:1,numSucceededTasks:2," + + "{otherInfo: {vertexName:'v1', numTasks:5,numFailedTasks:1,numSucceededTasks:2," + "numKilledTasks:3,numCompletedTasks:3}}," + - "{otherinfo: {vertexName:'v2',numTasks:10,numFailedTasks:1,numSucceededTasks:5," + + "{otherInfo: {vertexName:'v2',numTasks:10,numFailedTasks:1,numSucceededTasks:5," + "numKilledTasks:3,numCompletedTasks:4}}" + "]}"; @@ -151,7 +151,7 @@ public void testGetVertexStatusSimple() throws JSONException, TezException, IOEx Set statusOptions = new HashSet(1); statusOptions.add(StatusGetOpts.GET_COUNTERS); - final String jsonData = "{entities:[ {otherinfo:{numFailedTasks:1,numSucceededTasks:2," + + final String jsonData = "{entities:[ {otherInfo:{numFailedTasks:1,numSucceededTasks:2," + "status:'SUCCEEDED', vertexName:'vertex1name', numTasks:4, numKilledTasks: 3, " + "numCompletedTasks: 4, diagnostics: 'diagnostics1', " + "counters: { counterGroups: [ " + diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java b/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java index 8c8d2abb49..d0ebcaf0dc 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java @@ -52,10 +52,10 @@ public void testPseudoAuthenticatorConnectionUrlShouldHaveUserName() throws Exce ConnectionConfigurator connConf = mock(ConnectionConfigurator.class); TimelineReaderPseudoAuthenticatedStrategy.PseudoAuthenticatedURLConnectionFactory connectionFactory = new TimelineReaderPseudoAuthenticatedStrategy - .PseudoAuthenticatedURLConnectionFactory(connConf); + .PseudoAuthenticatedURLConnectionFactory(connConf); String inputUrl = "http://host:8080/path"; String expectedUrl = inputUrl + "?user.name=" + UserGroupInformation.getCurrentUser().getShortUserName(); - HttpURLConnection httpURLConnection = connectionFactory.getHttpURLConnection(new URL(inputUrl)); + HttpURLConnection httpURLConnection = connectionFactory.getConnection(new URL(inputUrl)); Assert.assertEquals(expectedUrl, httpURLConnection.getURL().toString()); } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java b/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java index c32f847e59..34f90fb656 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java @@ -133,7 +133,7 @@ public static JSONObject convertToJson(HistoryEvent historyEvent) throws JSONExc private static JSONObject convertDAGRecoveredEvent(DAGRecoveredEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, + jsonObject.put(ATSConstants.ENTITY_ID, event.getDagID().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name()); @@ -168,7 +168,7 @@ private static JSONObject convertDAGRecoveredEvent(DAGRecoveredEvent event) private static JSONObject convertAppLaunchedEvent(AppLaunchedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, + jsonObject.put(ATSConstants.ENTITY_ID, "tez_" + event.getApplicationId().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_APPLICATION.name()); @@ -186,7 +186,7 @@ private static JSONObject convertAppLaunchedEvent(AppLaunchedEvent event) throws private static JSONObject convertAMLaunchedEvent(AMLaunchedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, + jsonObject.put(ATSConstants.ENTITY_ID, "tez_" + event.getApplicationAttemptId().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_APPLICATION_ATTEMPT.name()); @@ -194,12 +194,12 @@ private static JSONObject convertAMLaunchedEvent(AMLaunchedEvent event) throws J // Related Entities JSONArray relatedEntities = new JSONArray(); JSONObject appEntity = new JSONObject(); - appEntity.put(ATSConstants.ENTITY, + appEntity.put(ATSConstants.ENTITY_ID, event.getApplicationAttemptId().getApplicationId().toString()); appEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.APPLICATION_ID); JSONObject appAttemptEntity = new JSONObject(); - appAttemptEntity.put(ATSConstants.ENTITY, + appAttemptEntity.put(ATSConstants.ENTITY_ID, event.getApplicationAttemptId().toString()); appAttemptEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.APPLICATION_ATTEMPT_ID); @@ -227,7 +227,7 @@ private static JSONObject convertAMLaunchedEvent(AMLaunchedEvent event) throws J private static JSONObject convertAMStartedEvent(AMStartedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, + jsonObject.put(ATSConstants.ENTITY_ID, "tez_" + event.getApplicationAttemptId().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_APPLICATION_ATTEMPT.name()); @@ -235,12 +235,12 @@ private static JSONObject convertAMStartedEvent(AMStartedEvent event) throws JSO // Related Entities JSONArray relatedEntities = new JSONArray(); JSONObject appEntity = new JSONObject(); - appEntity.put(ATSConstants.ENTITY, + appEntity.put(ATSConstants.ENTITY_ID, event.getApplicationAttemptId().getApplicationId().toString()); appEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.APPLICATION_ID); JSONObject appAttemptEntity = new JSONObject(); - appAttemptEntity.put(ATSConstants.ENTITY, + appAttemptEntity.put(ATSConstants.ENTITY_ID, event.getApplicationAttemptId().toString()); appAttemptEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.APPLICATION_ATTEMPT_ID); @@ -262,20 +262,20 @@ private static JSONObject convertAMStartedEvent(AMStartedEvent event) throws JSO private static JSONObject convertContainerLaunchedEvent(ContainerLaunchedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, + jsonObject.put(ATSConstants.ENTITY_ID, "tez_" + event.getContainerId().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_CONTAINER_ID.name()); JSONArray relatedEntities = new JSONArray(); JSONObject appAttemptEntity = new JSONObject(); - appAttemptEntity.put(ATSConstants.ENTITY, + appAttemptEntity.put(ATSConstants.ENTITY_ID, event.getApplicationAttemptId().toString()); appAttemptEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_APPLICATION_ATTEMPT.name()); JSONObject containerEntity = new JSONObject(); - containerEntity.put(ATSConstants.ENTITY, event.getContainerId().toString()); + containerEntity.put(ATSConstants.ENTITY_ID, event.getContainerId().toString()); containerEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.CONTAINER_ID); relatedEntities.put(appAttemptEntity); @@ -301,20 +301,20 @@ private static JSONObject convertContainerLaunchedEvent(ContainerLaunchedEvent e private static JSONObject convertContainerStoppedEvent(ContainerStoppedEvent event) throws JSONException { // structure is identical to ContainerLaunchedEvent JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, + jsonObject.put(ATSConstants.ENTITY_ID, "tez_" + event.getContainerId().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_CONTAINER_ID.name()); JSONArray relatedEntities = new JSONArray(); JSONObject appAttemptEntity = new JSONObject(); - appAttemptEntity.put(ATSConstants.ENTITY, + appAttemptEntity.put(ATSConstants.ENTITY_ID, event.getApplicationAttemptId().toString()); appAttemptEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_APPLICATION_ATTEMPT.name()); JSONObject containerEntity = new JSONObject(); - containerEntity.put(ATSConstants.ENTITY, event.getContainerId().toString()); + containerEntity.put(ATSConstants.ENTITY_ID, event.getContainerId().toString()); containerEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.CONTAINER_ID); relatedEntities.put(appAttemptEntity); @@ -343,7 +343,7 @@ private static JSONObject convertContainerStoppedEvent(ContainerStoppedEvent eve private static JSONObject convertDAGFinishedEvent(DAGFinishedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, + jsonObject.put(ATSConstants.ENTITY_ID, event.getDAGID().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name()); @@ -386,7 +386,7 @@ private static JSONObject convertDAGFinishedEvent(DAGFinishedEvent event) throws private static JSONObject convertDAGInitializedEvent(DAGInitializedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, + jsonObject.put(ATSConstants.ENTITY_ID, event.getDAGID().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name()); @@ -418,7 +418,7 @@ private static JSONObject convertDAGInitializedEvent(DAGInitializedEvent event) private static JSONObject convertDAGStartedEvent(DAGStartedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, + jsonObject.put(ATSConstants.ENTITY_ID, event.getDAGID().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name()); @@ -441,7 +441,7 @@ private static JSONObject convertDAGStartedEvent(DAGStartedEvent event) throws J private static JSONObject convertDAGSubmittedEvent(DAGSubmittedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, + jsonObject.put(ATSConstants.ENTITY_ID, event.getDAGID().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name()); @@ -449,27 +449,27 @@ private static JSONObject convertDAGSubmittedEvent(DAGSubmittedEvent event) thro // Related Entities JSONArray relatedEntities = new JSONArray(); JSONObject tezAppEntity = new JSONObject(); - tezAppEntity.put(ATSConstants.ENTITY, + tezAppEntity.put(ATSConstants.ENTITY_ID, "tez_" + event.getApplicationAttemptId().getApplicationId().toString()); tezAppEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_APPLICATION.name()); JSONObject tezAppAttemptEntity = new JSONObject(); - tezAppAttemptEntity.put(ATSConstants.ENTITY, + tezAppAttemptEntity.put(ATSConstants.ENTITY_ID, "tez_" + event.getApplicationAttemptId().toString()); tezAppAttemptEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_APPLICATION_ATTEMPT.name()); JSONObject appEntity = new JSONObject(); - appEntity.put(ATSConstants.ENTITY, + appEntity.put(ATSConstants.ENTITY_ID, event.getApplicationAttemptId().getApplicationId().toString()); appEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.APPLICATION_ID); JSONObject appAttemptEntity = new JSONObject(); - appAttemptEntity.put(ATSConstants.ENTITY, + appAttemptEntity.put(ATSConstants.ENTITY_ID, event.getApplicationAttemptId().toString()); appAttemptEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.APPLICATION_ATTEMPT_ID); JSONObject userEntity = new JSONObject(); - userEntity.put(ATSConstants.ENTITY, + userEntity.put(ATSConstants.ENTITY_ID, event.getUser()); userEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.USER); @@ -531,7 +531,7 @@ private static JSONObject convertDAGSubmittedEvent(DAGSubmittedEvent event) thro private static JSONObject convertTaskAttemptFinishedEvent(TaskAttemptFinishedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getTaskAttemptID().toString()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getTaskAttemptID().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_TASK_ATTEMPT_ID.name()); @@ -590,22 +590,22 @@ private static JSONObject convertTaskAttemptFinishedEvent(TaskAttemptFinishedEve private static JSONObject convertTaskAttemptStartedEvent(TaskAttemptStartedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getTaskAttemptID().toString()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getTaskAttemptID().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_TASK_ATTEMPT_ID.name()); // Related entities JSONArray relatedEntities = new JSONArray(); JSONObject nodeEntity = new JSONObject(); - nodeEntity.put(ATSConstants.ENTITY, event.getNodeId().toString()); + nodeEntity.put(ATSConstants.ENTITY_ID, event.getNodeId().toString()); nodeEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.NODE_ID); JSONObject containerEntity = new JSONObject(); - containerEntity.put(ATSConstants.ENTITY, event.getContainerId().toString()); + containerEntity.put(ATSConstants.ENTITY_ID, event.getContainerId().toString()); containerEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.CONTAINER_ID); JSONObject taskEntity = new JSONObject(); - taskEntity.put(ATSConstants.ENTITY, event.getTaskID().toString()); + taskEntity.put(ATSConstants.ENTITY_ID, event.getTaskID().toString()); taskEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_TASK_ID.name()); relatedEntities.put(nodeEntity); @@ -633,7 +633,7 @@ private static JSONObject convertTaskAttemptStartedEvent(TaskAttemptStartedEvent private static JSONObject convertTaskFinishedEvent(TaskFinishedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getTaskID().toString()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getTaskID().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_TASK_ID.name()); // Events @@ -664,13 +664,13 @@ private static JSONObject convertTaskFinishedEvent(TaskFinishedEvent event) thro private static JSONObject convertTaskStartedEvent(TaskStartedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getTaskID().toString()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getTaskID().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_TASK_ID.name()); // Related entities JSONArray relatedEntities = new JSONArray(); JSONObject vertexEntity = new JSONObject(); - vertexEntity.put(ATSConstants.ENTITY, event.getVertexID().toString()); + vertexEntity.put(ATSConstants.ENTITY_ID, event.getVertexID().toString()); vertexEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name()); relatedEntities.put(vertexEntity); jsonObject.put(ATSConstants.RELATED_ENTITIES, relatedEntities); @@ -696,7 +696,7 @@ private static JSONObject convertTaskStartedEvent(TaskStartedEvent event) throws private static JSONObject convertVertexFinishedEvent(VertexFinishedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getVertexID().toString()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getVertexID().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name()); // Events @@ -737,7 +737,7 @@ private static JSONObject convertVertexFinishedEvent(VertexFinishedEvent event) private static JSONObject convertVertexReconfigureDoneEvent(VertexConfigurationDoneEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getVertexID().toString()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getVertexID().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name()); // Events @@ -772,13 +772,13 @@ private static JSONObject convertVertexReconfigureDoneEvent(VertexConfigurationD private static JSONObject convertVertexInitializedEvent(VertexInitializedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getVertexID().toString()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getVertexID().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name()); // Related entities JSONArray relatedEntities = new JSONArray(); JSONObject vertexEntity = new JSONObject(); - vertexEntity.put(ATSConstants.ENTITY, event.getDAGID().toString()); + vertexEntity.put(ATSConstants.ENTITY_ID, event.getDAGID().toString()); vertexEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name()); relatedEntities.put(vertexEntity); jsonObject.put(ATSConstants.RELATED_ENTITIES, relatedEntities); @@ -812,13 +812,13 @@ private static JSONObject convertVertexInitializedEvent(VertexInitializedEvent e private static JSONObject convertVertexStartedEvent(VertexStartedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getVertexID().toString()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getVertexID().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name()); // Related entities JSONArray relatedEntities = new JSONArray(); JSONObject vertexEntity = new JSONObject(); - vertexEntity.put(ATSConstants.ENTITY, event.getDAGID().toString()); + vertexEntity.put(ATSConstants.ENTITY_ID, event.getDAGID().toString()); vertexEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name()); relatedEntities.put(vertexEntity); jsonObject.put(ATSConstants.RELATED_ENTITIES, relatedEntities); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java b/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java index fa3d734ba3..0d37a137e3 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java @@ -231,7 +231,7 @@ public void testConvertVertexReconfigureDoneEvent() throws JSONException { JSONObject jsonObject = HistoryEventJsonConversion.convertToJson(event); Assert.assertNotNull(jsonObject); - Assert.assertEquals(vId.toString(), jsonObject.getString(ATSConstants.ENTITY)); + Assert.assertEquals(vId.toString(), jsonObject.getString(ATSConstants.ENTITY_ID)); Assert.assertEquals(ATSConstants.TEZ_VERTEX_ID, jsonObject.get(ATSConstants.ENTITY_TYPE)); JSONArray events = jsonObject.getJSONArray(ATSConstants.EVENTS); diff --git a/tez-plugins/tez-history-parser/pom.xml b/tez-plugins/tez-history-parser/pom.xml index d50a16dabb..9e074f2bd8 100644 --- a/tez-plugins/tez-history-parser/pom.xml +++ b/tez-plugins/tez-history-parser/pom.xml @@ -148,8 +148,9 @@ test - com.sun.jersey - jersey-json + org.glassfish.jersey.media + jersey-media-json-jackson + test org.apache.hadoop diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/ATSImportTool.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/ATSImportTool.java index 5ed9d376c2..c98f627002 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/ATSImportTool.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/ATSImportTool.java @@ -28,12 +28,17 @@ import java.net.URISyntaxException; import java.net.URL; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.Iterator; import java.util.Objects; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.DefaultParser; @@ -42,7 +47,6 @@ import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.io.IOUtils; -import org.apache.commons.io.LineIterator; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.http.HttpConfig; @@ -60,20 +64,12 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Strings; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientHandlerException; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.UniformInterfaceException; -import com.sun.jersey.api.client.WebResource; -import com.sun.jersey.api.client.config.ClientConfig; -import com.sun.jersey.api.client.config.DefaultClientConfig; -import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory; -import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; -import com.sun.jersey.json.impl.provider.entity.JSONRootElementProvider; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; +import org.glassfish.jersey.client.ClientConfig; +import org.glassfish.jersey.client.HttpUrlConnectorProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -169,7 +165,7 @@ private void download() throws Exception { throw e; } finally { if (httpClient != null) { - httpClient.destroy(); + httpClient.close(); } IOUtils.closeQuietly(fos); } @@ -279,72 +275,64 @@ private void downloadJSONArrayFromATS(String url, ZipOutputStream zos, String ta //Set the last item in entities as the fromId url = baseUrl + "&fromId=" - + entities.getJSONObject(entities.length() - 1).getString(Constants.ENTITY); + + entities.getJSONObject(entities.length() - 1).getString(Constants.ENTITY_ID); - String firstItem = entities.getJSONObject(0).getString(Constants.ENTITY); - String lastItem = entities.getJSONObject(entities.length() - 1).getString(Constants.ENTITY); + String firstItem = entities.getJSONObject(0).getString(Constants.ENTITY_ID); + String lastItem = entities.getJSONObject(entities.length() - 1).getString(Constants.ENTITY_ID); LOG.info("Downloaded={}, First item={}, LastItem={}, new url={}", downloadedCount, firstItem, lastItem, url); } } - private void logErrorMessage(ClientResponse response) throws IOException { - LOG.error("Response status={}", response.getClientResponseStatus().toString()); - LineIterator it = null; + private void logErrorMessage(Response response) { + LOG.info("Response status={}", Integer.toString(response.getStatus())); try { - it = IOUtils.lineIterator(response.getEntityInputStream(), UTF8); - while (it.hasNext()) { - String line = it.nextLine(); - LOG.error(line); - } - } finally { - if (it != null) { - it.close(); + String entity = response.readEntity(String.class); + if (entity != null) { + LOG.info("Response entity={}", entity); } + } catch (Exception ignore) { + LOG.debug("Failed to read error response entity", ignore); } } //For secure cluster, this should work as long as valid ticket is available in the node. private JSONObject getJsonRootEntity(String url) throws TezException, IOException { try { - WebResource wr = getHttpClient().resource(url); - ClientResponse response = wr.accept(MediaType.APPLICATION_JSON_TYPE) - .type(MediaType.APPLICATION_JSON_TYPE) - .get(ClientResponse.class); + WebTarget target = getHttpClient().target(url); + Response response = target.request(MediaType.APPLICATION_JSON_TYPE) + .accept(MediaType.APPLICATION_JSON_TYPE) + .get(); - if (response.getClientResponseStatus() != ClientResponse.Status.OK) { + if (response.getStatus() != Response.Status.OK.getStatusCode()) { // In the case of secure cluster, if there is any auth exception it sends the data back as // a html page and JSON parsing could throw exceptions. Instead, get the stream contents // completely and log it in case of error. logErrorMessage(response); throw new TezException("Failed to get response from YARN Timeline: url: " + url); } - return response.getEntity(JSONObject.class); - } catch (ClientHandlerException e) { + String json = response.readEntity(String.class); + return new JSONObject(json); + } catch (Exception e) { throw new TezException("Error processing response from YARN Timeline. URL=" + url, e); - } catch (UniformInterfaceException e) { - throw new TezException("Error accessing content from YARN Timeline - unexpected response. " - + "URL=" + url, e); - } catch (IllegalArgumentException e) { - throw new TezException("Error accessing content from YARN Timeline - invalid url. URL=" + url, - e); } } private Client getHttpClient() { if (httpClient == null) { - ClientConfig config = new DefaultClientConfig(JSONRootElementProvider.App.class); - HttpURLConnectionFactory urlFactory = new PseudoAuthenticatedURLConnectionFactory(); - return new Client(new URLConnectionClientHandler(urlFactory), config); + ClientConfig config = new ClientConfig(); + config.connectorProvider(new HttpUrlConnectorProvider() + .connectionFactory(new PseudoAuthenticatedURLConnectionFactory())); + return ClientBuilder.newClient(config); } return httpClient; } - static class PseudoAuthenticatedURLConnectionFactory implements HttpURLConnectionFactory { + static class PseudoAuthenticatedURLConnectionFactory implements HttpUrlConnectorProvider.ConnectionFactory { @Override - public HttpURLConnection getHttpURLConnection(URL url) throws IOException { + public HttpURLConnection getConnection(URL url) throws IOException { String tokenString = (url.getQuery() == null ? "?" : "&") + "user.name=" + - URLEncoder.encode(UserGroupInformation.getCurrentUser().getShortUserName(), "UTF8"); + URLEncoder.encode(UserGroupInformation.getCurrentUser().getShortUserName(), StandardCharsets.UTF_8); return (HttpURLConnection) (new URL(url.toString() + tokenString)).openConnection(); } } diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java index b26184ad06..8279c6974d 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java @@ -180,7 +180,7 @@ protected void postProcessMaps(Map vertexJsonMap, if (relatedEntities == null) { //This can happen when CONTAINER_EXITED abruptly. (e.g Container failed, exitCode=1) LOG.debug("entity {} did not have related entities", - jsonObject.optJSONObject(Constants.ENTITY)); + jsonObject.optJSONObject(Constants.ENTITY_ID)); } else { JSONObject subJsonObject = relatedEntities.optJSONObject(0); if (subJsonObject != null) { @@ -188,7 +188,7 @@ protected void postProcessMaps(Map vertexJsonMap, if (!Strings.isNullOrEmpty(nodeId) && nodeId.equalsIgnoreCase(Constants.NODE_ID)) { //populate it in otherInfo JSONObject otherInfo = jsonObject.optJSONObject(Constants.OTHER_INFO); - String nodeIdVal = subJsonObject.optString(Constants.ENTITY); + String nodeIdVal = subJsonObject.optString(Constants.ENTITY_ID); if (otherInfo != null && nodeIdVal != null) { otherInfo.put(Constants.NODE_ID, nodeIdVal); } @@ -202,7 +202,7 @@ protected void postProcessMaps(Map vertexJsonMap, .equalsIgnoreCase(Constants.CONTAINER_ID)) { //populate it in otherInfo JSONObject otherInfo = jsonObject.optJSONObject(Constants.OTHER_INFO); - String containerIdVal = subJsonObject.optString(Constants.ENTITY); + String containerIdVal = subJsonObject.optString(Constants.ENTITY_ID); if (otherInfo != null && containerIdVal != null) { otherInfo.put(Constants.CONTAINER_ID, containerIdVal); } @@ -225,7 +225,7 @@ protected void readEventsFromSource(String dagId, JSONObjectSource source, while (source.hasNext()) { JSONObject jsonObject = source.next(); - String entity = jsonObject.getString(Constants.ENTITY); + String entity = jsonObject.getString(Constants.ENTITY_ID); String entityType = jsonObject.getString(Constants.ENTITY_TYPE); switch (entityType) { case Constants.TEZ_DAG_ID: @@ -255,7 +255,7 @@ protected void readEventsFromSource(String dagId, JSONObjectSource source, JSONObject subEntity = relatedEntities.getJSONObject(i); String subEntityType = subEntity.optString(Constants.ENTITY_TYPE); if (subEntityType != null && subEntityType.equals(Constants.USER)) { - userName = subEntity.getString(Constants.ENTITY); + userName = subEntity.getString(Constants.ENTITY_ID); break; } } diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java index 802b189cf6..13e04e323b 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java @@ -102,7 +102,7 @@ public class DagInfo extends BaseInfo { Preconditions.checkArgument(jsonObject.getString(Constants.ENTITY_TYPE).equalsIgnoreCase (Constants.TEZ_DAG_ID)); - dagId = StringInterner.intern(jsonObject.getString(Constants.ENTITY)); + dagId = StringInterner.intern(jsonObject.getString(Constants.ENTITY_ID)); //Parse additional Info JSONObject otherInfoNode = jsonObject.getJSONObject(Constants.OTHER_INFO); diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskAttemptInfo.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskAttemptInfo.java index f88918a1e6..b2e9583218 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskAttemptInfo.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskAttemptInfo.java @@ -98,7 +98,7 @@ public String getTaskAttemptId() { jsonObject.getString(Constants.ENTITY_TYPE).equalsIgnoreCase (Constants.TEZ_TASK_ATTEMPT_ID)); - taskAttemptId = StringInterner.intern(jsonObject.optString(Constants.ENTITY)); + taskAttemptId = StringInterner.intern(jsonObject.optString(Constants.ENTITY_ID)); //Parse additional Info final JSONObject otherInfoNode = jsonObject.getJSONObject(Constants.OTHER_INFO); diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskInfo.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskInfo.java index 7810bd61f8..819d6908ca 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskInfo.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskInfo.java @@ -73,7 +73,7 @@ public class TaskInfo extends BaseInfo { jsonObject.getString(Constants.ENTITY_TYPE).equalsIgnoreCase (Constants.TEZ_TASK_ID)); - taskId = StringInterner.intern(jsonObject.optString(Constants.ENTITY)); + taskId = StringInterner.intern(jsonObject.optString(Constants.ENTITY_ID)); //Parse additional Info final JSONObject otherInfoNode = jsonObject.getJSONObject(Constants.OTHER_INFO); diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/VertexInfo.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/VertexInfo.java index bde42d17d5..b061d81839 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/VertexInfo.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/VertexInfo.java @@ -94,7 +94,7 @@ public class VertexInfo extends BaseInfo { jsonObject.getString(Constants.ENTITY_TYPE).equalsIgnoreCase (Constants.TEZ_VERTEX_ID)); - vertexId = StringInterner.intern(jsonObject.optString(Constants.ENTITY)); + vertexId = StringInterner.intern(jsonObject.optString(Constants.ENTITY_ID)); taskInfoMap = Maps.newHashMap(); inEdgeList = Lists.newLinkedList(); diff --git a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/HistoryEventProtoJsonConversion.java b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/HistoryEventProtoJsonConversion.java index 2172447f86..f2bb209fdd 100644 --- a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/HistoryEventProtoJsonConversion.java +++ b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/HistoryEventProtoJsonConversion.java @@ -115,7 +115,7 @@ public static JSONObject convertToJson(HistoryEventProto historyEvent) throws JS private static JSONObject convertDAGRecoveredEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getDagId()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getDagId()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name()); // Related Entities not needed as should have been done in @@ -142,7 +142,7 @@ private static JSONObject convertDAGRecoveredEvent(HistoryEventProto event) thro private static JSONObject convertAppLaunchedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, "tez_" + event.getAppId().toString()); + jsonObject.put(ATSConstants.ENTITY_ID, "tez_" + event.getAppId().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_APPLICATION.name()); // Other info to tag with Tez App @@ -157,16 +157,16 @@ private static JSONObject convertAppLaunchedEvent(HistoryEventProto event) throw private static JSONObject convertAMLaunchedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, "tez_" + event.getAppAttemptId().toString()); + jsonObject.put(ATSConstants.ENTITY_ID, "tez_" + event.getAppAttemptId().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_APPLICATION_ATTEMPT.name()); // Related Entities JSONArray relatedEntities = new JSONArray(); JSONObject appEntity = new JSONObject(); - appEntity.put(ATSConstants.ENTITY, event.getAppId().toString()); + appEntity.put(ATSConstants.ENTITY_ID, event.getAppId().toString()); appEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.APPLICATION_ID); JSONObject appAttemptEntity = new JSONObject(); - appAttemptEntity.put(ATSConstants.ENTITY, event.getAppAttemptId().toString()); + appAttemptEntity.put(ATSConstants.ENTITY_ID, event.getAppAttemptId().toString()); appAttemptEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.APPLICATION_ATTEMPT_ID); relatedEntities.put(appEntity); relatedEntities.put(appAttemptEntity); @@ -192,16 +192,16 @@ private static JSONObject convertAMLaunchedEvent(HistoryEventProto event) throws private static JSONObject convertAMStartedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, "tez_" + event.getAppAttemptId().toString()); + jsonObject.put(ATSConstants.ENTITY_ID, "tez_" + event.getAppAttemptId().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_APPLICATION_ATTEMPT.name()); // Related Entities JSONArray relatedEntities = new JSONArray(); JSONObject appEntity = new JSONObject(); - appEntity.put(ATSConstants.ENTITY, event.getAppId().toString()); + appEntity.put(ATSConstants.ENTITY_ID, event.getAppId().toString()); appEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.APPLICATION_ID); JSONObject appAttemptEntity = new JSONObject(); - appAttemptEntity.put(ATSConstants.ENTITY, event.getAppAttemptId().toString()); + appAttemptEntity.put(ATSConstants.ENTITY_ID, event.getAppAttemptId().toString()); appAttemptEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.APPLICATION_ATTEMPT_ID); relatedEntities.put(appEntity); relatedEntities.put(appAttemptEntity); @@ -222,17 +222,17 @@ private static JSONObject convertAMStartedEvent(HistoryEventProto event) throws private static JSONObject convertContainerLaunchedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, + jsonObject.put(ATSConstants.ENTITY_ID, "tez_" + getDataValueByKey(event, ATSConstants.CONTAINER_ID)); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_CONTAINER_ID.name()); JSONArray relatedEntities = new JSONArray(); JSONObject appAttemptEntity = new JSONObject(); - appAttemptEntity.put(ATSConstants.ENTITY, event.getAppAttemptId().toString()); + appAttemptEntity.put(ATSConstants.ENTITY_ID, event.getAppAttemptId().toString()); appAttemptEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_APPLICATION_ATTEMPT.name()); JSONObject containerEntity = new JSONObject(); - containerEntity.put(ATSConstants.ENTITY, getDataValueByKey(event, ATSConstants.CONTAINER_ID)); + containerEntity.put(ATSConstants.ENTITY_ID, getDataValueByKey(event, ATSConstants.CONTAINER_ID)); containerEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.CONTAINER_ID); relatedEntities.put(appAttemptEntity); @@ -258,17 +258,17 @@ private static JSONObject convertContainerStoppedEvent(HistoryEventProto event) throws JSONException { // structure is identical to ContainerLaunchedEvent JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, + jsonObject.put(ATSConstants.ENTITY_ID, "tez_" + getDataValueByKey(event, ATSConstants.CONTAINER_ID)); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_CONTAINER_ID.name()); JSONArray relatedEntities = new JSONArray(); JSONObject appAttemptEntity = new JSONObject(); - appAttemptEntity.put(ATSConstants.ENTITY, event.getAppAttemptId().toString()); + appAttemptEntity.put(ATSConstants.ENTITY_ID, event.getAppAttemptId().toString()); appAttemptEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_APPLICATION_ATTEMPT.name()); JSONObject containerEntity = new JSONObject(); - containerEntity.put(ATSConstants.ENTITY, getDataValueByKey(event, ATSConstants.CONTAINER_ID)); + containerEntity.put(ATSConstants.ENTITY_ID, getDataValueByKey(event, ATSConstants.CONTAINER_ID)); containerEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.CONTAINER_ID); relatedEntities.put(appAttemptEntity); @@ -297,7 +297,7 @@ private static JSONObject convertContainerStoppedEvent(HistoryEventProto event) private static JSONObject convertDAGFinishedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getDagId()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getDagId()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name()); // Related Entities not needed as should have been done in @@ -341,7 +341,7 @@ private static JSONObject convertDAGFinishedEvent(HistoryEventProto event) throw private static JSONObject convertDAGInitializedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getDagId()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getDagId()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name()); // Related Entities not needed as should have been done in @@ -364,7 +364,7 @@ private static JSONObject convertDAGInitializedEvent(HistoryEventProto event) private static JSONObject convertDAGStartedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getDagId()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getDagId()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name()); // Related Entities not needed as should have been done in @@ -384,25 +384,25 @@ private static JSONObject convertDAGStartedEvent(HistoryEventProto event) throws private static JSONObject convertDAGSubmittedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getDagId()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getDagId()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name()); // Related Entities JSONArray relatedEntities = new JSONArray(); JSONObject tezAppEntity = new JSONObject(); - tezAppEntity.put(ATSConstants.ENTITY, "tez_" + event.getAppId().toString()); + tezAppEntity.put(ATSConstants.ENTITY_ID, "tez_" + event.getAppId().toString()); tezAppEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_APPLICATION.name()); JSONObject tezAppAttemptEntity = new JSONObject(); - tezAppAttemptEntity.put(ATSConstants.ENTITY, "tez_" + event.getAppAttemptId().toString()); + tezAppAttemptEntity.put(ATSConstants.ENTITY_ID, "tez_" + event.getAppAttemptId().toString()); tezAppAttemptEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_APPLICATION_ATTEMPT.name()); JSONObject appEntity = new JSONObject(); - appEntity.put(ATSConstants.ENTITY, event.getAppId().toString()); + appEntity.put(ATSConstants.ENTITY_ID, event.getAppId().toString()); appEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.APPLICATION_ID); JSONObject appAttemptEntity = new JSONObject(); - appAttemptEntity.put(ATSConstants.ENTITY, event.getAppAttemptId().toString()); + appAttemptEntity.put(ATSConstants.ENTITY_ID, event.getAppAttemptId().toString()); appAttemptEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.APPLICATION_ATTEMPT_ID); JSONObject userEntity = new JSONObject(); - userEntity.put(ATSConstants.ENTITY, event.getUser()); + userEntity.put(ATSConstants.ENTITY_ID, event.getUser()); userEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.USER); relatedEntities.put(tezAppEntity); @@ -452,7 +452,7 @@ private static JSONObject convertDAGSubmittedEvent(HistoryEventProto event) thro private static JSONObject convertTaskAttemptFinishedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getTaskAttemptId()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getTaskAttemptId()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_TASK_ATTEMPT_ID.name()); // Events @@ -503,21 +503,21 @@ private static JSONObject convertTaskAttemptFinishedEvent(HistoryEventProto even private static JSONObject convertTaskAttemptStartedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getTaskAttemptId()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getTaskAttemptId()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_TASK_ATTEMPT_ID.name()); // Related entities JSONArray relatedEntities = new JSONArray(); JSONObject nodeEntity = new JSONObject(); - nodeEntity.put(ATSConstants.ENTITY, getDataValueByKey(event, ATSConstants.NODE_ID)); + nodeEntity.put(ATSConstants.ENTITY_ID, getDataValueByKey(event, ATSConstants.NODE_ID)); nodeEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.NODE_ID); JSONObject containerEntity = new JSONObject(); - containerEntity.put(ATSConstants.ENTITY, getDataValueByKey(event, ATSConstants.CONTAINER_ID)); + containerEntity.put(ATSConstants.ENTITY_ID, getDataValueByKey(event, ATSConstants.CONTAINER_ID)); containerEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.CONTAINER_ID); JSONObject taskEntity = new JSONObject(); - taskEntity.put(ATSConstants.ENTITY, event.getTaskAttemptId()); + taskEntity.put(ATSConstants.ENTITY_ID, event.getTaskAttemptId()); taskEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_TASK_ID.name()); relatedEntities.put(nodeEntity); @@ -546,7 +546,7 @@ private static JSONObject convertTaskAttemptStartedEvent(HistoryEventProto event private static JSONObject convertTaskFinishedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getTaskId()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getTaskId()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_TASK_ID.name()); // Events @@ -577,13 +577,13 @@ private static JSONObject convertTaskFinishedEvent(HistoryEventProto event) thro private static JSONObject convertTaskStartedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getTaskId()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getTaskId()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_TASK_ID.name()); // Related entities JSONArray relatedEntities = new JSONArray(); JSONObject vertexEntity = new JSONObject(); - vertexEntity.put(ATSConstants.ENTITY, event.getVertexId()); + vertexEntity.put(ATSConstants.ENTITY_ID, event.getVertexId()); vertexEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name()); relatedEntities.put(vertexEntity); jsonObject.put(ATSConstants.RELATED_ENTITIES, relatedEntities); @@ -610,7 +610,7 @@ private static JSONObject convertTaskStartedEvent(HistoryEventProto event) throw private static JSONObject convertVertexFinishedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getVertexId()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getVertexId()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name()); // Events @@ -653,7 +653,7 @@ private static JSONObject convertVertexFinishedEvent(HistoryEventProto event) private static JSONObject convertVertexReconfigureDoneEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getVertexId()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getVertexId()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name()); // Events @@ -681,13 +681,13 @@ private static JSONObject convertVertexReconfigureDoneEvent(HistoryEventProto ev private static JSONObject convertVertexInitializedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getVertexId()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getVertexId()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name()); // Related entities JSONArray relatedEntities = new JSONArray(); JSONObject vertexEntity = new JSONObject(); - vertexEntity.put(ATSConstants.ENTITY, event.getDagId()); + vertexEntity.put(ATSConstants.ENTITY_ID, event.getDagId()); vertexEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name()); relatedEntities.put(vertexEntity); jsonObject.put(ATSConstants.RELATED_ENTITIES, relatedEntities); @@ -720,13 +720,13 @@ private static JSONObject convertVertexInitializedEvent(HistoryEventProto event) private static JSONObject convertVertexStartedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getVertexId()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getVertexId()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name()); // Related entities JSONArray relatedEntities = new JSONArray(); JSONObject vertexEntity = new JSONObject(); - vertexEntity.put(ATSConstants.ENTITY, event.getDagId()); + vertexEntity.put(ATSConstants.ENTITY_ID, event.getDagId()); vertexEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name()); relatedEntities.put(vertexEntity); jsonObject.put(ATSConstants.RELATED_ENTITIES, relatedEntities); diff --git a/tez-plugins/tez-yarn-timeline-history-with-acls/pom.xml b/tez-plugins/tez-yarn-timeline-history-with-acls/pom.xml index 7505e71c83..1037c68d4f 100644 --- a/tez-plugins/tez-yarn-timeline-history-with-acls/pom.xml +++ b/tez-plugins/tez-yarn-timeline-history-with-acls/pom.xml @@ -140,8 +140,8 @@ test - com.sun.jersey - jersey-json + org.glassfish.jersey.media + jersey-media-json-jackson test diff --git a/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java b/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java index f701b6eaa7..36e6008f2f 100644 --- a/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java +++ b/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java @@ -31,7 +31,11 @@ import java.util.List; import java.util.Random; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -46,6 +50,7 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.tez.client.TezClient; +import org.apache.tez.common.ATSConstants; import org.apache.tez.common.ReflectionUtils; import org.apache.tez.common.security.DAGAccessControls; import org.apache.tez.dag.api.DAG; @@ -66,9 +71,6 @@ import org.apache.tez.tests.MiniTezClusterWithTimeline; import com.google.common.collect.Sets; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.WebResource; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; @@ -148,23 +150,38 @@ public static void tearDown() throws InterruptedException { // To be replaced after Timeline has java APIs for domains private K getTimelineData(String url, Class clazz) { - Client client = new Client(); - WebResource resource = client.resource(url); - - ClientResponse response = resource.accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + Client client = ClientBuilder.newClient(); + WebTarget target = client.target(url); + Response response = target.request(MediaType.APPLICATION_JSON).get(); assertEquals(200, response.getStatus()); - assertTrue(MediaType.APPLICATION_JSON_TYPE.isCompatible(response.getType())); - - JSONObject entity = response.getEntity(JSONObject.class); - K converted = null; + assertTrue(MediaType.APPLICATION_JSON_TYPE.isCompatible(response.getMediaType())); + String entityStr = response.readEntity(String.class); try { - converted = convertJSONObjectToTimelineObject(entity, clazz); + JSONObject jsonObject = new JSONObject(entityStr); + JSONObject effectiveJson; + // Handle the nesting introduced by Jersey 2/Jackson + if (clazz == TimelineDomain.class) { + // TimelineDomain is annotated @XmlRootElement(name="domain"), Jersey 2/Jackson wraps it + effectiveJson = jsonObject.has("domain") + ? jsonObject.getJSONObject("domain") + : jsonObject; + } else if (clazz == TimelineEntity.class) { + // TimelineEntity is annotated @XmlRootElement(name="entity"), Jersey 2/Jackson wrap it + effectiveJson = jsonObject.has("entity") + ? jsonObject.getJSONObject("entity") + : jsonObject; + } else { + throw new RuntimeException("Unsupported class for JSON conversion: " + clazz); + } + K converted = convertJSONObjectToTimelineObject(effectiveJson, clazz); + assertNotNull(converted); + return converted; } catch (JSONException e) { throw new RuntimeException(e); + } finally { + response.close(); + client.close(); } - assertNotNull(converted); - return converted; } private K convertJSONObjectToTimelineObject(JSONObject jsonObj, Class clazz) throws JSONException { @@ -178,9 +195,9 @@ private K convertJSONObjectToTimelineObject(JSONObject jsonObj, Class cla return (K) domain; } else if (clazz == TimelineEntity.class) { TimelineEntity entity = new TimelineEntity(); - entity.setEntityId(jsonObj.getString("entity")); - entity.setEntityType(jsonObj.getString("entitytype")); - entity.setDomainId(jsonObj.getString("domain")); + entity.setEntityId(jsonObj.getString(ATSConstants.ENTITY_ID)); + entity.setEntityType(jsonObj.getString(ATSConstants.ENTITY_TYPE)); + entity.setDomainId(jsonObj.getString(ATSConstants.DOMAIN_ID)); entity.setEvents(getEventsFromJSON(jsonObj)); return (K) entity; } else { @@ -194,7 +211,7 @@ private List getEventsFromJSON(JSONObject jsonObj) throws JSONExc JSONArray arrEvents = jsonObj.getJSONArray("events"); for (int i = 0; i < arrEvents.length(); i++) { TimelineEvent event = new TimelineEvent(); - event.setEventType(((JSONObject) arrEvents.get(i)).getString("eventtype")); + event.setEventType(((JSONObject) arrEvents.get(i)).getString(ATSConstants.EVENT_TYPE)); events.add(event); } return events; @@ -453,12 +470,12 @@ public void testDagLoggingDisabled() throws Exception { historyLoggingService.handle(new DAGHistoryEvent(tezDAGID, submittedEvent)); Thread.sleep(1000l); String url = "http://" + timelineAddress + "/ws/v1/timeline/TEZ_DAG_ID/"+event.getDAGID(); - Client client = new Client(); - WebResource resource = client.resource(url); - - ClientResponse response = resource.accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + Client client = ClientBuilder.newClient(); + WebTarget target = client.target(url); + Response response = target.request(MediaType.APPLICATION_JSON).get(); assertEquals(404, response.getStatus()); + response.close(); + client.close(); } /** @@ -498,17 +515,18 @@ public void testDagLoggingEnabled() throws Exception { historyLoggingService.handle(new DAGHistoryEvent(tezDAGID, submittedEvent)); Thread.sleep(1000l); String url = "http://" + timelineAddress + "/ws/v1/timeline/TEZ_DAG_ID/"+event.getDAGID(); - Client client = new Client(); - WebResource resource = client.resource(url); - - ClientResponse response = resource.accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + Client client = ClientBuilder.newClient(); + WebTarget target = client.target(url); + Response response = target.request(MediaType.APPLICATION_JSON).get(); assertEquals(200, response.getStatus()); - assertTrue(MediaType.APPLICATION_JSON_TYPE.isCompatible(response.getType())); - JSONObject entityJson = response.getEntity(JSONObject.class); + assertTrue(MediaType.APPLICATION_JSON_TYPE.isCompatible(response.getMediaType())); + String entityStr = response.readEntity(String.class); + JSONObject entityJson = new JSONObject(entityStr); TimelineEntity entity = convertJSONObjectToTimelineObject(entityJson, TimelineEntity.class); assertEquals(entity.getEntityType(), "TEZ_DAG_ID"); assertEquals(entity.getEvents().get(0).getEventType(), HistoryEventType.DAG_SUBMITTED.toString()); + response.close(); + client.close(); } private static final String atsHistoryACLManagerClassName = diff --git a/tez-plugins/tez-yarn-timeline-history-with-fs/pom.xml b/tez-plugins/tez-yarn-timeline-history-with-fs/pom.xml index 55ed04bbc7..8390ba0922 100644 --- a/tez-plugins/tez-yarn-timeline-history-with-fs/pom.xml +++ b/tez-plugins/tez-yarn-timeline-history-with-fs/pom.xml @@ -148,8 +148,8 @@ test - com.sun.jersey - jersey-json + org.glassfish.jersey.media + jersey-media-json-jackson test diff --git a/tez-plugins/tez-yarn-timeline-history/pom.xml b/tez-plugins/tez-yarn-timeline-history/pom.xml index 8406500110..0acd8faf4b 100644 --- a/tez-plugins/tez-yarn-timeline-history/pom.xml +++ b/tez-plugins/tez-yarn-timeline-history/pom.xml @@ -134,8 +134,8 @@ test - com.sun.jersey - jersey-json + org.glassfish.jersey.media + jersey-media-json-jackson test diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryWithMiniCluster.java b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryWithMiniCluster.java index 1ff3bfd8d7..27b0da059d 100644 --- a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryWithMiniCluster.java +++ b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryWithMiniCluster.java @@ -21,7 +21,11 @@ import java.io.IOException; import java.util.Random; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -40,10 +44,6 @@ import org.apache.tez.runtime.library.processor.SleepProcessor.SleepProcessorConfig; import org.apache.tez.tests.MiniTezClusterWithTimeline; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.WebResource; - import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -115,16 +115,18 @@ public static void tearDown() throws InterruptedException { // To be replaced after Timeline has java APIs for domains private K getTimelineData(String url, Class clazz) { - Client client = new Client(); - WebResource resource = client.resource(url); + Client client = ClientBuilder.newClient(); + WebTarget target = client.target(url); - ClientResponse response = resource.accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + Response response = target.request(MediaType.APPLICATION_JSON) + .get(); Assert.assertEquals(200, response.getStatus()); - Assert.assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + Assert.assertTrue(MediaType.APPLICATION_JSON_TYPE.isCompatible(response.getMediaType())); - K entity = response.getEntity(clazz); + K entity = response.readEntity(clazz); Assert.assertNotNull(entity); + response.close(); + client.close(); return entity; } diff --git a/tez-tools/analyzers/job-analyzer/pom.xml b/tez-tools/analyzers/job-analyzer/pom.xml index 34f10b3757..6953332af0 100644 --- a/tez-tools/analyzers/job-analyzer/pom.xml +++ b/tez-tools/analyzers/job-analyzer/pom.xml @@ -157,8 +157,9 @@ test - com.sun.jersey - jersey-json + org.glassfish.jersey.media + jersey-media-json-jackson + test org.apache.hadoop diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/utils/Utils.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/utils/Utils.java index c95049544f..ab667133c3 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/utils/Utils.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/utils/Utils.java @@ -23,14 +23,14 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import javax.annotation.Nullable; + import org.apache.tez.dag.utils.Graph; import org.apache.tez.history.parser.datamodel.AdditionalInputOutputDetails; import org.apache.tez.history.parser.datamodel.DagInfo; import org.apache.tez.history.parser.datamodel.EdgeInfo; import org.apache.tez.history.parser.datamodel.VertexInfo; -import com.sun.istack.Nullable; - public final class Utils { private static Pattern sanitizeLabelPattern = Pattern.compile("[:\\-\\W]+");