/** * Copyright 2015 StreamSets Inc. * * Licensed under the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.streamsets.datacollector.restapi; import com.google.common.collect.ImmutableList; import com.streamsets.datacollector.bundles.SupportBundleManager; import com.streamsets.datacollector.event.handler.remote.RemoteEventHandlerTask; import com.streamsets.datacollector.io.DataStore; import com.streamsets.datacollector.main.RuntimeInfo; import com.streamsets.datacollector.main.UserGroupManager; import com.streamsets.datacollector.restapi.bean.BeanHelper; import com.streamsets.datacollector.restapi.bean.DPMInfoJson; import com.streamsets.datacollector.restapi.bean.MultiStatusResponseJson; import com.streamsets.datacollector.restapi.bean.SupportBundleContentDefinitionJson; import com.streamsets.datacollector.restapi.bean.UserJson; import com.streamsets.datacollector.store.PipelineStoreException; import com.streamsets.datacollector.util.AuthzRole; import com.streamsets.datacollector.util.Configuration; import com.streamsets.lib.security.http.RemoteSSOService; import com.streamsets.lib.security.http.SSOConstants; import com.streamsets.lib.security.http.SSOPrincipal; import com.streamsets.pipeline.api.impl.Utils; import com.streamsets.pipeline.lib.util.ThreadUtil; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.Authorization; import org.apache.commons.configuration2.PropertiesConfiguration; import org.apache.commons.configuration2.builder.FileBasedConfigurationBuilder; import org.apache.commons.configuration2.builder.fluent.Parameters; import org.apache.commons.configuration2.convert.DefaultListDelimiterHandler; import org.apache.commons.configuration2.ex.ConfigurationException; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.glassfish.jersey.client.filter.CsrfProtectionFilter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.security.DenyAll; import javax.annotation.security.RolesAllowed; import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.Consumes; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; import javax.ws.rs.client.ClientBuilder; import javax.ws.rs.client.Entity; import javax.ws.rs.core.Context; import javax.ws.rs.core.GenericType; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.lang.management.ManagementFactory; import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @Path("/v1/system") @Api(value = "system") @DenyAll public class AdminResource { private static final Logger LOG = LoggerFactory.getLogger(AdminResource.class); private static final String APP_TOKEN_FILE = "application-token.txt"; private static final String APP_TOKEN_FILE_PROP_VAL = "@application-token.txt@"; private final RuntimeInfo runtimeInfo; private final Configuration config; private final UserGroupManager userGroupManager; private final SupportBundleManager supportBundleManager; @Inject public AdminResource( RuntimeInfo runtimeInfo, Configuration config, UserGroupManager userGroupManager, SupportBundleManager supportBundleManager ) { this.runtimeInfo = runtimeInfo; this.config = config; this.userGroupManager = userGroupManager; this.supportBundleManager = supportBundleManager; } @POST @Path("/shutdown") @ApiOperation(value = "Shutdown SDC", authorizations = @Authorization(value = "basic")) @Produces(MediaType.APPLICATION_JSON) @RolesAllowed({AuthzRole.ADMIN, AuthzRole.ADMIN_REMOTE}) public Response shutdown() throws PipelineStoreException { Thread thread = new Thread("Shutdown Request") { @Override public void run() { // sleeping 500ms to allow the HTTP response to go back ThreadUtil.sleep(500); runtimeInfo.shutdown(0); } }; thread.setDaemon(true); thread.start(); return Response.ok().build(); } @POST @Path("/restart") @ApiOperation(value = "Restart SDC", authorizations = @Authorization(value = "basic")) @Produces(MediaType.APPLICATION_JSON) @RolesAllowed({AuthzRole.ADMIN, AuthzRole.ADMIN_REMOTE}) public Response restart() throws PipelineStoreException { Thread thread = new Thread("Shutdown Request") { @Override public void run() { // sleeping 500ms to allow the HTTP response to go back ThreadUtil.sleep(500); runtimeInfo.shutdown(88); } }; thread.setDaemon(true); thread.start(); return Response.ok().build(); } @POST @Path("/enableDPM") @ApiOperation( value = "Enables DPM by getting application token from DPM", authorizations = @Authorization(value = "basic") ) @Produces(MediaType.APPLICATION_JSON) @RolesAllowed({AuthzRole.ADMIN, AuthzRole.ADMIN_REMOTE}) public Response enableDPM(DPMInfoJson dpmInfo) throws IOException { Utils.checkNotNull(dpmInfo, "DPMInfo"); String dpmBaseURL = dpmInfo.getBaseURL(); if (dpmBaseURL.endsWith("/")) { dpmBaseURL = dpmBaseURL.substring(0, dpmBaseURL.length() - 1); } // Since we support enabling/Disabling DPM, first check if token already exists for the given DPM URL. // If token exists skip first 3 steps String currentDPMBaseURL = config.get(RemoteSSOService.DPM_BASE_URL_CONFIG, ""); String currentAppAuthToken = config.get(RemoteSSOService.SECURITY_SERVICE_APP_AUTH_TOKEN_CONFIG, "").trim(); if (!currentDPMBaseURL.equals(dpmBaseURL) || currentAppAuthToken.length() == 0) { // 1. Login to DPM to get user auth token Response response = null; try { Map<String, String> loginJson = new HashMap<>(); loginJson.put("userName", dpmInfo.getUserID()); loginJson.put("password", dpmInfo.getUserPassword()); response = ClientBuilder.newClient() .target(dpmBaseURL + "/security/public-rest/v1/authentication/login") .register(new CsrfProtectionFilter("CSRF")) .request() .post(Entity.json(loginJson)); if (response.getStatus() != Response.Status.OK.getStatusCode()) { throw new RuntimeException(Utils.format("DPM Login failed, status code '{}': {}", response.getStatus(), response.readEntity(String.class) )); } } finally { if (response != null) { response.close(); } } String userAuthToken = response.getHeaderString(SSOConstants.X_USER_AUTH_TOKEN); String appAuthToken = null; // 2. Create Data Collector application token try { Map<String, Object> newComponentJson = new HashMap<>(); newComponentJson.put("organization", dpmInfo.getOrganization()); newComponentJson.put("componentType", "dc"); newComponentJson.put("numberOfComponents", 1); newComponentJson.put("active", true); response = ClientBuilder.newClient() .target(dpmBaseURL + "/security/rest/v1/organization/" + dpmInfo.getOrganization() + "/components") .register(new CsrfProtectionFilter("CSRF")) .request() .header(SSOConstants.X_USER_AUTH_TOKEN, userAuthToken) .put(Entity.json(newComponentJson)); if (response.getStatus() != Response.Status.CREATED.getStatusCode()) { throw new RuntimeException(Utils.format("DPM Create Application Token failed, status code '{}': {}", response.getStatus(), response.readEntity(String.class) )); } List<Map<String, Object>> newComponent = response.readEntity(new GenericType<List<Map<String,Object>>>() {}); if (newComponent.size() > 0) { appAuthToken = (String) newComponent.get(0).get("fullAuthToken"); } else { throw new RuntimeException("DPM Create Application Token failed: No token data from DPM Server."); } } finally { if (response != null) { response.close(); } // Logout from DPM try { response = ClientBuilder.newClient() .target(dpmBaseURL + "/security/_logout") .register(new CsrfProtectionFilter("CSRF")) .request() .header(SSOConstants.X_USER_AUTH_TOKEN, userAuthToken) .cookie(SSOConstants.AUTHENTICATION_COOKIE_PREFIX + "LOGIN", userAuthToken) .get(); } finally { if (response != null) { response.close(); } } } // 3. Update App Token file DataStore dataStore = new DataStore(new File(runtimeInfo.getConfigDir(), APP_TOKEN_FILE)); try (OutputStream os = dataStore.getOutputStream()) { IOUtils.write(appAuthToken, os); dataStore.commit(os); } finally { dataStore.release(); } } // 4. Update dpm.properties file try { FileBasedConfigurationBuilder<PropertiesConfiguration> builder = new FileBasedConfigurationBuilder<>(PropertiesConfiguration.class) .configure(new Parameters().properties() .setFileName(runtimeInfo.getConfigDir() + "/dpm.properties") .setThrowExceptionOnMissing(true) .setListDelimiterHandler(new DefaultListDelimiterHandler(';')) .setIncludesAllowed(false)); PropertiesConfiguration config = null; config = builder.getConfiguration(); config.setProperty(RemoteSSOService.DPM_ENABLED, "true"); config.setProperty(RemoteSSOService.DPM_BASE_URL_CONFIG, dpmBaseURL); config.setProperty(RemoteSSOService.SECURITY_SERVICE_APP_AUTH_TOKEN_CONFIG, APP_TOKEN_FILE_PROP_VAL); if (dpmInfo.getLabels() != null && dpmInfo.getLabels().size() > 0) { config.setProperty(RemoteEventHandlerTask.REMOTE_JOB_LABELS, StringUtils.join(dpmInfo.getLabels(), ',')); } else { config.setProperty(RemoteEventHandlerTask.REMOTE_JOB_LABELS, ""); } builder.save(); } catch (ConfigurationException e) { throw new RuntimeException(Utils.format("Updating dpm.properties file failed: {}", e.getMessage()), e); } return Response.ok().build(); } @POST @Path("/disableDPM") @ApiOperation( value = "Disables DPM", authorizations = @Authorization(value = "basic") ) @Produces(MediaType.APPLICATION_JSON) @RolesAllowed({AuthzRole.ADMIN, AuthzRole.ADMIN_REMOTE}) public Response disableDPM(@Context HttpServletRequest request) throws IOException { // check if DPM enabled if (!runtimeInfo.isDPMEnabled()) { throw new RuntimeException("disableDPM is supported only when DPM is enabled"); } String dpmBaseURL = config.get(RemoteSSOService.DPM_BASE_URL_CONFIG, ""); if (dpmBaseURL.endsWith("/")) { dpmBaseURL = dpmBaseURL.substring(0, dpmBaseURL.length() - 1); } // 1. Get DPM user auth token from request cookie SSOPrincipal ssoPrincipal = (SSOPrincipal)request.getUserPrincipal(); String userAuthToken = ssoPrincipal.getTokenStr(); String organizationId = ssoPrincipal.getOrganizationId(); String componentId = runtimeInfo.getId(); // 2. Deactivate Data Collector System Component Response response = null; try { response = ClientBuilder.newClient() .target(dpmBaseURL + "/security/rest/v1/organization/" + organizationId + "/components/deactivate") .register(new CsrfProtectionFilter("CSRF")) .request() .header(SSOConstants.X_USER_AUTH_TOKEN, userAuthToken) .header(SSOConstants.X_REST_CALL, true) .post(Entity.json(ImmutableList.of(componentId))); if (response.getStatus() != Response.Status.OK.getStatusCode()) { throw new RuntimeException(Utils.format( " Deactivate Data Collector System Component from DPM failed, status code '{}': {}", response.getStatus(), response.readEntity(String.class) )); } } finally { if (response != null) { response.close(); } } // 3. Delete Data Collector System Component try { response = ClientBuilder.newClient() .target(dpmBaseURL + "/security/rest/v1/organization/" + organizationId + "/components/delete") .register(new CsrfProtectionFilter("CSRF")) .request() .header(SSOConstants.X_USER_AUTH_TOKEN, userAuthToken) .header(SSOConstants.X_REST_CALL, true) .post(Entity.json(ImmutableList.of(componentId))); if (response.getStatus() != Response.Status.OK.getStatusCode()) { throw new RuntimeException(Utils.format( " Deactivate Data Collector System Component from DPM failed, status code '{}': {}", response.getStatus(), response.readEntity(String.class) )); } } finally { if (response != null) { response.close(); } } // 4. Delete from Job Runner SDC list try { response = ClientBuilder.newClient() .target(dpmBaseURL + "/jobrunner/rest/v1/sdc/" + componentId) .register(new CsrfProtectionFilter("CSRF")) .request() .header(SSOConstants.X_USER_AUTH_TOKEN, userAuthToken) .header(SSOConstants.X_REST_CALL, true) .delete(); if (response.getStatus() != Response.Status.OK.getStatusCode()) { throw new RuntimeException(Utils.format( "Delete from DPM Job Runner SDC list failed, status code '{}': {}", response.getStatus(), response.readEntity(String.class) )); } } finally { if (response != null) { response.close(); } } // 5. Update App Token file DataStore dataStore = new DataStore(new File(runtimeInfo.getConfigDir(), APP_TOKEN_FILE)); try (OutputStream os = dataStore.getOutputStream()) { IOUtils.write("", os); dataStore.commit(os); } finally { dataStore.release(); } // 4. Update dpm.properties file try { FileBasedConfigurationBuilder<PropertiesConfiguration> builder = new FileBasedConfigurationBuilder<>(PropertiesConfiguration.class) .configure(new Parameters().properties() .setFileName(runtimeInfo.getConfigDir() + "/dpm.properties") .setThrowExceptionOnMissing(true) .setListDelimiterHandler(new DefaultListDelimiterHandler(';')) .setIncludesAllowed(false)); PropertiesConfiguration config = null; config = builder.getConfiguration(); config.setProperty(RemoteSSOService.DPM_ENABLED, "false"); builder.save(); } catch (ConfigurationException e) { throw new RuntimeException(Utils.format("Updating dpm.properties file failed: {}", e.getMessage()), e); } return Response.ok().build(); } @POST @Path("/createDPMUsers") @ApiOperation( value = "Create Users & Groups in DPM", authorizations = @Authorization(value = "basic") ) @Produces(MediaType.APPLICATION_JSON) @RolesAllowed({AuthzRole.ADMIN, AuthzRole.ADMIN_REMOTE}) public Response createDPMUsers(DPMInfoJson dpmInfo) throws IOException { Utils.checkNotNull(dpmInfo, "DPMInfo"); List<String> successEntities = new ArrayList<>(); List<String> errorMessages = new ArrayList<>(); String dpmBaseURL = dpmInfo.getBaseURL(); if (dpmBaseURL.endsWith("/")) { dpmBaseURL = dpmBaseURL.substring(0, dpmBaseURL.length() - 1); } // 1. Login to DPM to get user auth token Response response = null; try { Map<String, String> loginJson = new HashMap<>(); loginJson.put("userName", dpmInfo.getUserID()); loginJson.put("password", dpmInfo.getUserPassword()); response = ClientBuilder.newClient() .target(dpmBaseURL + "/security/public-rest/v1/authentication/login") .register(new CsrfProtectionFilter("CSRF")) .request() .post(Entity.json(loginJson)); if (response.getStatus() != Response.Status.OK.getStatusCode()) { throw new RuntimeException(Utils.format("DPM Login failed, status code '{}': {}", response.getStatus(), response.readEntity(String.class) )); } } finally { if (response != null) { response.close(); } } String userAuthToken = response.getHeaderString(SSOConstants.X_USER_AUTH_TOKEN); String appAuthToken = null; // 2. Create Groups for (Map<String, Object> group: dpmInfo.getDpmGroupList()) { try { response = ClientBuilder.newClient() .target(dpmBaseURL + "/security/rest/v1/organization/" + dpmInfo.getOrganization() + "/groups") .register(new CsrfProtectionFilter("CSRF")) .request() .header(SSOConstants.X_USER_AUTH_TOKEN, userAuthToken) .put(Entity.json(group)); if (response.getStatus() != Response.Status.CREATED.getStatusCode()) { errorMessages.add(Utils.format("DPM Create Group '{}' failed, status code '{}': {}", group.get("id"), response.getStatus(), response.readEntity(String.class) )); } else { successEntities.add(Utils.format("Created DPM Group '{}' successfully", group.get("id"))); } } finally { if (response != null) { response.close(); } } } // 3. Create Users for (Map<String, Object> user: dpmInfo.getDpmUserList()) { try { response = ClientBuilder.newClient() .target(dpmBaseURL + "/security/rest/v1/organization/" + dpmInfo.getOrganization() + "/users") .register(new CsrfProtectionFilter("CSRF")) .request() .header(SSOConstants.X_USER_AUTH_TOKEN, userAuthToken) .put(Entity.json(user)); if (response.getStatus() != Response.Status.CREATED.getStatusCode()) { errorMessages.add(Utils.format("DPM Create User '{}' failed, status code '{}': {}", user.get("id"), response.getStatus(), response.readEntity(String.class) )); } else { successEntities.add(Utils.format("Created DPM User '{}' successfully", user.get("id"))); } } finally { if (response != null) { response.close(); } } } // 4. Logout from DPM try { response = ClientBuilder.newClient() .target(dpmBaseURL + "/security/_logout") .register(new CsrfProtectionFilter("CSRF")) .request() .header(SSOConstants.X_USER_AUTH_TOKEN, userAuthToken) .cookie(SSOConstants.AUTHENTICATION_COOKIE_PREFIX + "LOGIN", userAuthToken) .get(); } finally { if (response != null) { response.close(); } } return Response.status(207) .type(MediaType.APPLICATION_JSON) .entity(new MultiStatusResponseJson<>(successEntities, errorMessages)).build(); } @GET @Path("/threads") @ApiOperation(value = "Returns Thread Dump along with stack trace", response = Map.class, responseContainer = "List", authorizations = @Authorization(value = "basic")) @Produces(MediaType.APPLICATION_JSON) @RolesAllowed({AuthzRole.ADMIN, AuthzRole.ADMIN_REMOTE}) public Response getThreadsDump() throws IOException { ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean(); ThreadInfo[] threads = threadMXBean.dumpAllThreads(true, true); List<Map> augmented = new ArrayList<>(threads.length); for (ThreadInfo thread : threads) { Map<String, Object> map = new LinkedHashMap<>(); map.put("threadInfo", thread); map.put("userTimeNanosecs", threadMXBean.getThreadUserTime(thread.getThreadId())); map.put("cpuTimeNanosecs", threadMXBean.getThreadCpuTime(thread.getThreadId())); augmented.add(map); } return Response.ok(augmented).build(); } @GET @Path("/directories") @ApiOperation(value = "Returns SDC Directories", response = Map.class, authorizations = @Authorization(value = "basic")) @Produces(MediaType.APPLICATION_JSON) @RolesAllowed({AuthzRole.ADMIN, AuthzRole.ADMIN_REMOTE}) public Response getSDCDirectories() throws IOException { Map<String, Object> map = new LinkedHashMap<>(); map.put("runtimeDir", runtimeInfo.getRuntimeDir()); map.put("configDir", runtimeInfo.getConfigDir()); map.put("dataDir", runtimeInfo.getDataDir()); map.put("logDir", runtimeInfo.getLogDir()); map.put("resourcesDir", runtimeInfo.getResourcesDir()); map.put("libsExtraDir", runtimeInfo.getLibsExtraDir()); return Response.ok(map).build(); } @GET @Path("/users") @ApiOperation( value = "Returns All Users Info", response = UserJson.class, responseContainer = "List", authorizations = @Authorization(value = "basic") ) @Produces(MediaType.APPLICATION_JSON) @RolesAllowed({ AuthzRole.ADMIN, AuthzRole.ADMIN_REMOTE, AuthzRole.CREATOR, AuthzRole.CREATOR_REMOTE }) public Response getUsers() throws IOException { return Response.ok(userGroupManager.getUsers()).build(); } @GET @Path("/groups") @ApiOperation( value = "Returns All Group names", response = UserJson.class, responseContainer = "List", authorizations = @Authorization(value = "basic") ) @Produces(MediaType.APPLICATION_JSON) @RolesAllowed({ AuthzRole.ADMIN, AuthzRole.ADMIN_REMOTE, AuthzRole.CREATOR, AuthzRole.CREATOR_REMOTE }) public Response getGroups() throws IOException { return Response.ok(userGroupManager.getGroups()).build(); } @GET @Path("/bundle/list") @ApiOperation( value = "Return list of available content generators for support bundles.", response = SupportBundleContentDefinitionJson.class, responseContainer = "List", authorizations = @Authorization(value = "basic") ) @Produces(MediaType.APPLICATION_JSON) @RolesAllowed({ AuthzRole.ADMIN, AuthzRole.ADMIN_REMOTE }) public Response getSupportBundlesContentGenerators() throws IOException { return Response.ok(BeanHelper.wrapSupportBundleDefinitions(supportBundleManager.getContentDefinitions())).build(); } @GET @Path("/bundle/generate") @ApiOperation( value = "Generates a new support bundle.", response = Object.class, authorizations = @Authorization(value = "basic") ) @Consumes(MediaType.APPLICATION_JSON) @Produces("application/octet-stream") @RolesAllowed({ AuthzRole.ADMIN, AuthzRole.ADMIN_REMOTE }) public Response createSupportBundlesContentGenerators( @QueryParam("generators") @DefaultValue("") String generators ) throws IOException { List<String> generatorList = Collections.emptyList(); if(!generators.isEmpty()) { generatorList = Arrays.asList(generators.split(",")); } SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd_HH:mm:ss"); StringBuilder builder = new StringBuilder("bundle_"); builder.append(runtimeInfo.getId()); builder.append("_"); builder.append(dateFormat.format(new Date())); builder.append(".zip"); return Response .ok() .header("content-disposition", "attachment; filename=\"" + builder.toString() + "\"") .entity(supportBundleManager.generateNewBundle(generatorList)) .build(); } }