Distributed tracing with Istio, Kubernetes, Elasticsearch, Fluentd and Kibana

Elasticseach

Kibana

Fluentd

Fluentd is an open source data collector for unified logging layer.

docker pull fluent/fluentd-kubernetes-daemonset:v1.14.5-debian-elasticsearch7-amd64–1.1

Distributed Tracing

Spring Boot specific implementation

@Service
public class FlowService {

private final ConcurrentMap<Thread, String> flowStates = new ConcurrentHashMap<>();
private final static String FlowIdName = "flow-id";public String getFlowId() {

Thread thread = Thread.currentThread();

return this.flowStates.getOrDefault(thread, null);
}

public void setFlowId(@NotNull String flowId) {

String thread = Thread.currentThread();
this.flowStates.put(thread, flowId);
this.injectFlowIdIntoLoggerContext(flowId);
}

public void clearFlowId() {

String thread = Thread.currentThread();

this.clearFlowIdFromLoggerContext();

this.flowStates.remove(thread);
}

private void injectFlowIdIntoLoggerContext(String flowId) {

MDC.put(FlowIdName, flowId);
}

private void clearFlowIdFromLoggerContext() {

MDC.remove(FlowIdName);
}
}
public class IncomingRequestFilter implements Filter {

private final FlowService flowService;

public IncomingRequestFilter(FlowService flowService) {
this.flowService= flowService;
}

@Override
public void doFilter(ServletRequest servletRequest, ServletResponse response, FilterChain chain) throws IOException, ServletException {

if (servletRequest instanceof HttpServletRequest) {
HttpServletRequest request = (HttpServletRequest) servletRequest;
String flowIdValue = request.getHeader("x-request-id");
if(!flowIdValue.isBlank())
this.flowService.setFlowId(flowIdValue);
}

chain.doFilter(servletRequest, response);

this.flowService.clearFlowId();
}
}
@Configuration
@ConditionalOnWebApplication
public class IncomingRequestFilterConfiguration {

@Bean
public FilterRegistrationBean injectInContextFilter(FlowService flowService) {
FilterRegistrationBean registrationBean = new FilterRegistrationBean();
registrationBean.setFilter(new IncomingRequestFilter(flowService));
registrationBean.setOrder(Ordered.HIGHEST_PRECEDENCE);
return registrationBean;
}
}
@Component
public class OutgoingInterceptor implements ClientHttpRequestInterceptor {

private FlowService flowService;

public OutgoingInterceptor(FlowService flowService) {
this.flowService= flowService;
}


@Override
public ClientHttpResponse intercept(HttpRequest request, byte[] body, ClientHttpRequestExecution execution) throws IOException {

String flowId = this.flowService.getFlowId();

if (!flowId.isBlank()) {

request.getHeaders().add("x-request-id", flowId);
}

return execution.execute(request, body);
}
}
@Configuration
public class RestTemplateConfiguration {

@Bean
@Primary
@ConditionalOnMissingBean(search = SearchStrategy.CURRENT)
public RestTemplate restTemplate() {
RestTemplate restTemplate = new RestTemplate();
List<ClientHttpRequestInterceptor> interceptors = restTemplate.getInterceptors();

if (CollectionUtils.isEmpty(interceptors)) {
interceptors = new ArrayList<>();
}

interceptors.add(this.tracingInterceptor);
restTemplate.setInterceptors(interceptors);
return restTemplate;
}
}
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<appender name="Console"
class="ch.qos.logback.core.ConsoleAppender">
<encoder class="net.logstash.logback.encoder.LogstashEncoder">
</encoder>
</appender>
<root level="INFO">
<appender-ref ref="Console" />
</root>
</configuration>

Querying Indexed Logs


flow-id : *

--

--

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store