Problem Addressed

In a service oriented architecture, known as SOA, we have several microservices that get regularly deployed. The  strategy and various needs of deployment of such services has already been solved by applications like kubernetes. From a quality perspective there needs to be a high level test after any service is deployed or any configuration change is made to that service. One legacy method is to have a bunch of manual testers validate that the service behavior is working as expected. A better alternative is to use make an endpoint that gets called in every service whenever a service receives load or a configuration update. This endpoint then runs a series of tests that will validate that the service is in a good condition. When it is done it reports back to the endpoint that was called the status. You can then consume this anyway you prefer. Log it, trigger alerts, and etc.

SmokeTestResult Class

This is pretty much a copy of DropWizard’s HealthCheck class, except we need this to be serializable so that Spring can send the object into JSON. Jackson needs a public constructor to do this because it determines the class structure from Java’s reflection.

package com.domo.bedrock.health;


import com.fasterxml.jackson.annotation.JsonIgnore;

/**
 * Created by derekcarr on 4/5/16.
 */
public abstract class SmokeTestCheck {

    protected String name;

    public abstract SmokeTestCheck.Result check() throws Exception;

    public SmokeTestCheck(String name) {
        this.name = name;
    }

    public String getName() {
        return name;
    }


    public static class Result {
        private static final Result HEALTHY = new Result(true, null, null);
        private static final int PRIME = 31;

        /**
         * Returns a healthy {@link Result} with no additional message.
         *
         * @return a healthy {@link Result} with no additional message
         */
        public static Result healthy() {
            return HEALTHY;
        }

        /**
         * Returns a healthy {@link Result} with an additional message.
         *
         * @param message an informative message
         * @return a healthy {@link Result} with an additional message
         */
        public static Result healthy(String message) {
            return new Result(true, message, null);
        }

        /**
         * Returns a healthy {@link Result} with a formatted message.
         * <p/>
         * Message formatting follows the same rules as {@link String#format(String, Object...)}.
         *
         * @param message a message format
         * @param args    the arguments apply to the message format
         * @return a healthy {@link Result} with an additional message
         * @see String#format(String, Object...)
         */
        public static Result healthy(String message, Object... args) {
            return healthy(String.format(message, args));
        }

        /**
         * Returns an unhealthy {@link Result} with the given message.
         *
         * @param message an informative message describing how the health check failed
         * @return an unhealthy {@link Result} with the given message
         */
        public static Result unhealthy(String message) {
            return new Result(false, message, null);
        }

        /**
         * Returns an unhealthy {@link Result} with a formatted message.
         * <p/>
         * Message formatting follows the same rules as {@link String#format(String, Object...)}.
         *
         * @param message a message format
         * @param args    the arguments apply to the message format
         * @return an unhealthy {@link Result} with an additional message
         * @see String#format(String, Object...)
         */
        public static Result unhealthy(String message, Object... args) {
            return unhealthy(String.format(message, args));
        }

        /**
         * Returns an unhealthy {@link Result} with the given error.
         *
         * @param error an exception thrown during the health check
         * @return an unhealthy {@link Result} with the given error
         */
        public static Result unhealthy(Throwable error) {
            return new Result(false, error.getMessage(), error);
        }

        private final boolean healthy;
        private final String message;
        private final Throwable error;

        public Result(boolean isHealthy, String message, Throwable error) {
            this.healthy = isHealthy;
            this.message = message;
            this.error = error;
        }

        /**
         * Returns {@code true} if the result indicates the component is healthy; {@code false}
         * otherwise.
         *
         * @return {@code true} if the result indicates the component is healthy
         */
        public boolean isHealthy() {
            return healthy;
        }

        /**
         * Returns any additional message for the result, or {@code null} if the result has no
         * message.
         *
         * @return any additional message for the result, or {@code null}
         */
        public String getMessage() {
            return message;
        }

        /**
         * Returns any exception for the result, or {@code null} if the result has no exception.
         *
         * @return any exception for the result, or {@code null}
         */
        @JsonIgnore
        public Throwable getError() {
            return error;
        }

        @Override
        public boolean equals(Object o) {
            if (this == o) {
                return true;
            }
            if (o == null || getClass() != o.getClass()) {
                return false;
            }
            final Result result = (Result) o;
            return healthy == result.healthy &&
                    !(error != null ? !error.equals(result.error) : result.error != null) &&
                    !(message != null ? !message.equals(result.message) : result.message != null);
        }

        @Override
        public int hashCode() {
            int result = (healthy ? 1 : 0);
            result = PRIME * result + (message != null ? message.hashCode() : 0);
            result = PRIME * result + (error != null ? error.hashCode() : 0);
            return result;
        }

        @Override
        public String toString() {
            final StringBuilder builder = new StringBuilder("Result{isHealthy=");
            builder.append(healthy);
            if (message != null) {
                builder.append(", message=").append(message);
            }
            if (error != null) {
                builder.append(", error=").append(error);
            }
            builder.append('}');
            return builder.toString();
        }
    }
}

The Smoketest Event

package com.domo.bedrock.service.event;

import com.domo.bedrock.health.SmokeTestCheck;
import org.springframework.context.ApplicationEvent;

import java.util.HashMap;
import java.util.Map;


public class SmokeTestEvent extends ApplicationEvent {
    private Map<String, SmokeTestCheck.Result> map;

    /**
     * Create a new ApplicationEvent.
     *
     * @param source the component that published the event (never {@code null})
     */
    public SmokeTestEvent(Map<String, SmokeTestCheck.Result> source) {
        super(source);
        this.map = source;
    }

    public Map<String, SmokeTestCheck.Result> getHealthCheckMap() {
        return new HashMap(this.map);
    }

    public void addHealthCheckResult(String id, SmokeTestCheck.Result result){
      map.put(id, result);
    }
}

The Endpoint each microservice will have

@RequestMapping(value = "/smoketest", method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
@ResponseBody
public Map<String, SmokeTestCheck.Result> requestSmokeTestHealthCheck() {
    Map<String, SmokeTestCheck.Result> map = new HashMap<>();
    SmokeTestEvent smokeTestEvent = new SmokeTestEvent(map);
    applicationContext.publishEvent(smokeTestEvent);
    return smokeTestEvent.getHealthCheckMap();
}

This will publish an event that the service will consume by implementing an application listener. When the listener is finished it will return all the results synchronously.

Default Application Listener

Simply extend this class and implement the method that returns the SmokeTest.Result information. This can run anything from simple ping/pong endpoint tests to a full behavioral test of the service.

package com.domo.bedrock.health;

import com.domo.bedrock.service.event.SmokeTestEvent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationListener;
import org.springframework.stereotype.Component;

import java.util.List;

/**
 * Created by derekcarr on 4/4/16.
 */
@Component
public abstract class SmokeTestListener implements ApplicationListener<SmokeTestEvent> {

    protected static final Logger LOGGER = LoggerFactory.getLogger(SmokeTestListener.class);

    @Autowired
    protected List<SmokeTestCheck> tests;

    @Override
    public abstract void onApplicationEvent(SmokeTestEvent event);

}

 Unit Test Endpoint With MockMVC

This has extra dependencies because of our deployment manager and preauthentication strategies, but this allows us to create a mock of the event and make sure that the endpoint returns what is expected. There is a better way to validate this – using json to do the validation, but we had conflicts with the various versions.

package com.domo.maestro.service;

import com.codahale.metrics.MetricRegistry;
import com.domo.bedrock.health.SmokeTestCheck;
import com.domo.bedrock.maestro.metrics.MaestroCustomMetricsAppender;
import com.domo.bedrock.service.CoreAutoConfiguration;
import com.domo.bedrock.service.ToeProvider;
import com.domo.bedrock.service.event.SmokeTestEvent;
import com.domo.bedrock.web.WebAutoConfiguration;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.web.HttpMessageConvertersAutoConfiguration;
import org.springframework.cloud.autoconfigure.RefreshAutoConfiguration;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationListener;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.support.PropertySourcesPlaceholderConfigurer;
import org.springframework.core.env.MutablePropertySources;
import org.springframework.core.env.PropertySource;
import org.springframework.mock.env.MockPropertySource;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.testng.AbstractTestNGSpringContextTests;
import org.springframework.test.context.web.WebAppConfiguration;
import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import org.springframework.web.context.WebApplicationContext;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;

import java.util.EmptyStackException;

import static org.hamcrest.Matchers.containsString;
import static org.mockito.Mockito.mock;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;

/**
 * Created by derekcarr on 3/31/16.
 */
@WebAppConfiguration
@ContextConfiguration(classes = {
        MaestroResourceTest.resourceTestConfig.class,
        CoreAutoConfiguration.class,
        HttpMessageConvertersAutoConfiguration.class,
        WebAutoConfiguration.class,
        RefreshAutoConfiguration.class
})
public class MaestroResourceTest extends AbstractTestNGSpringContextTests {

    @Configuration
    public static class resourceTestConfig {
        @Bean
        public MaestroResource maestroResource(ApplicationContext applicationContext1) {
            DefaultHandlerProvider defaultHandlerProvider = mock(DefaultHandlerProvider.class);
            MetricRegistry metricsRegistry = mock(MetricRegistry.class);
            ServiceContext serviceContext = mock(ServiceContext.class);
            MaestroCustomMetricsAppender maestroCustomMetricsAppender = mock(MaestroCustomMetricsAppender.class);
            return new MaestroResource(defaultHandlerProvider,metricsRegistry,serviceContext, maestroCustomMetricsAppender, applicationContext1);
        }

        @Bean
        public SmokeEventListener smokeEventListener() {
            return new SmokeEventListener();
        }

        @Bean
        public static PropertySourcesPlaceholderConfigurer placeHolderConfigurer() {
            PropertySourcesPlaceholderConfigurer pspc = new PropertySourcesPlaceholderConfigurer();
            MutablePropertySources propertySources = new MutablePropertySources();
            PropertySource ps = new MockPropertySource()
                    .withProperty("requireSecurityAdapters", "false")
                    .withProperty("authenticationKey", "hippopotomonstrosesquipedaliophobiahippopotomonstrosesquipedaliophobia")
                    .withProperty("bedrock.spring.hmac.enabled","false");
            propertySources.addFirst(ps);
            pspc.setPropertySources(propertySources);
            return pspc;
        }

        @Bean
        public ToeProvider toeProvider() {
            return mock(ToeProvider.class);
        }

    }


    public static class SmokeEventListener implements ApplicationListener<SmokeTestEvent> {
        @Override
        public void onApplicationEvent(SmokeTestEvent event) {
            event.addHealthCheckResult("Test all the awesome things!", SmokeTestCheck.Result.healthy());
            event.addHealthCheckResult("Something Terrible has happened", SmokeTestCheck.Result.unhealthy("There are no more cookies in the break room."));
            event.addHealthCheckResult("Throw your hands Up - they're playing Clint's Song", SmokeTestCheck.Result.unhealthy(new EmptyStackException()));
        }
    }

    @Autowired
    private WebApplicationContext wac;
    private MockMvc mockMvc;

    private String expected = ",\"Something Terrible has happened\":{\"healthy\":false,\"message\":\"There are no more cookies in the break room.\"},\"Throw your hands Up - they're playing Clint's Song\":{\"healthy\":false,\"message\":null}}";

    @BeforeClass
    public void setupSpec() {
        mockMvc = MockMvcBuilders
                .webAppContextSetup(this.wac)
                .build();
    }

    @Test
    /** test for {@link MaestroResource#requestSmokeTestHealthCheck()} */
    public void test_smoketest_endpoint() throws Exception {
        mockMvc.perform(get("/maestro/smoketest"))
                // HTTP 200 returned
                .andDo(print())
                .andExpect(status().isOk())
                .andExpect(content().string(containsString("Test all the awesome things!\":{\"healthy\":true,\"message\":null}")))
                .andExpect(content().string(containsString("Something Terrible has happened\":{\"healthy\":false,\"message\":\"There are no more cookies in the break room.\"}")))
                .andExpect(content().string(containsString("Throw your hands Up - they're playing Clint's Song\":{\"healthy\":false,\"message\":null}")));
    }

}