A Whirlwind Tour of Testing in Elixir 🚂

Daniel Caixinha

Software Engineer @ onfido.png

gitlab.png github.png elixir_forum.png slack.png @dcaixinha

elixir_conf_eu.png

Why?

  • Tests are an integral part of any application
  • Carelessly built test suites will slow you down
  • Redundant coverage will kill morale
  • Testing isn't just about catching bugs
  • Different types of testing complement each other
  • Help build better test suites
elixir_conf_eu.png

The testing pyramid!

testing_pyramid_bar_low.png

elixir_conf_eu.png

The testing pyramid!

testing_pyramid_bar_high.png

elixir_conf_eu.png

The testing pyramid!

testing_pyramid_curved.png

elixir_conf_eu.png

The testing pyramid!

testing_pyramid_unit.png

elixir_conf_eu.png

Some "universal" rules

It's not a unit test if:

  • it talks to the database
  • it communicates across the network
  • it touches the file system
  • it can't run at the same time as any of your other unit tests
  • you have to do special things to your environment to run it.
  • – Michael Feathers
elixir_conf_eu.png

Some "universal" rules

It's not a unit test if:

  • it talks to the database
  • it communicates across the network
  • it touches the file system
  • it can't run at the same time as any of your other unit tests
  • you have to do special things to your environment to run it.
  • – Michael Feathers
elixir_conf_eu.png

Detroit-school TDD (a.k.a. classical)
vs.
London-school TDD (a.k.a. mockist)

elixir_conf_eu.png

unit_testing.png

elixir_conf_eu.png

unit_testing2.png

elixir_conf_eu.png

defmodule Demo.Validators.AddressValidator do
  @callback validation_level() :: String.p
  @callback set_validation_level(String.p) :: :ok
  @callback validate(String.p) :: boolean

  def validation_level() do
    Application.get_env(:demo, :validation_level)
  end

  def set_validation_level(level) do
    Application.put_env(:demo, :validation_level, level)
  end

  def validate(address) do
    :ok = emit_event("address_validated")

    address
    |> String.length()
    |> validate_length()
  end

  defp validate_length(address_length) when address_length < 32, do: true
  defp validate_length(_address_length), do: false

  defp emit_event(event_description), do: Demo.Events.EventEmitter.emit(event_description)
end
elixir_conf_eu.png
defmodule Demo.Validators.AddressValidatorTest do
  use ExUnit.Case, async: true
  alias Demo.Validators.AddressValidator, as: Subject

  describe "when the address is fewer than 32 chars" do
    test "it validates the address" do
      assert Subject.validate("valid address") == true
    end
  end

  describe "when the address is longer than 32 chars" do
    test "it invalidates the address" do
      assert Subject.validate("some very very very long address") == false
    end
  end
end
elixir_conf_eu.png

unit_testing3.png

elixir_conf_eu.png
defmodule Demo.Validators.AddressValidator do
  @callback validation_level() :: String.p
  @callback set_validation_level(String.p) :: :ok
  @callback validate(String.p) :: boolean

  def validation_level() do
    Application.get_env(:demo, :validation_level)
  end

  def set_validation_level(level) do
    Application.put_env(:demo, :validation_level, level)
  end

  def validate(address) do
    :ok = emit_event("address_validated")

    address
    |> String.length()
    |> validate_length()
  end

  defp validate_length(address_length) when address_length < 32, do: true
  defp validate_length(_address_length), do: false

  defp emit_event(event_description), do: Demo.Events.EventEmitter.emit(event_description)
end
elixir_conf_eu.png
test "it sets the validation level to the provided parameter" do
  Subject.set_validation_level("some_validation_level")

  assert Subject.validation_level() == "some_validation_level"
end
elixir_conf_eu.png

unit_testing4.png

elixir_conf_eu.png

defmodule Demo.Validators.AddressValidator do
  @callback validation_level() :: String.p
  @callback set_validation_level(String.p) :: :ok
  @callback validate(String.p) :: boolean

  def validation_level() do
    Application.get_env(:demo, :validation_level)
  end

  def set_validation_level(level) do
    Application.put_env(:demo, :validation_level, level)
  end

  def validate(address) do
    :ok = emit_event("address_validated")

    address
    |> String.length()
    |> validate_length()
  end

  defp validate_length(address_length) when address_length < 32, do: true
  defp validate_length(_address_length), do: false

  defp emit_event(event_description), do: Demo.Events.EventEmitter.emit(event_description)
end
elixir_conf_eu.png
defmodule Demo.Validators.AddressValidator do
  @callback validation_level() :: String.p
  @callback set_validation_level(String.p) :: :ok
  @callback validate(String.p) :: boolean

  @event_emitter Application.get_env(:demo, :event_emitter)

  def validation_level() do
    Application.get_env(:demo, :validation_level)
  end

  def set_validation_level(level) do
    Application.put_env(:demo, :validation_level, level)
  end

  def validate(address) do
    :ok = emit_event("address_validated")

    address
    |> String.length()
    |> validate_length()
  end

  defp validate_length(address_length) when address_length < 32, do: true
  defp validate_length(_address_length), do: false

  defp emit_event(event_description), do: @event_emitter.emit(event_description)
end
elixir_conf_eu.png
# config/config.exs
use Mix.Config

config :demo,
  event_emitter: Demo.Events.EventEmitter
elixir_conf_eu.png
# config/test.exs
use Mix.Config

config :demo,
  event_emitter: Demo.Events.EventEmitterMock
elixir_conf_eu.png

defmodule Demo.Events.EventEmitterMock do
  @callback emit(String.t) :: :ok

  def emit("address_validated") do
    send(self(), :address_validated_event_emitted)
    :ok
  end
end
elixir_conf_eu.png

test "it emits the :address_validated event" do
  Subject.validate("test address")

  assert_receive :address_validated_event_emitted
end
elixir_conf_eu.png

Over time, the mock may (will?) diverge from the real thing

elixir_conf_eu.png

Meanwhile in production…

this_is_fine.jpg

elixir_conf_eu.png

Enter Mox! 🕺

elixir_conf_eu.png

mox.png

elixir_conf_eu.png

mocks_blogpost.png

elixir_conf_eu.png

# mix.exs
defp deps do
  [
    # ...
    {:mox, "~> 0.5", only: :test}
  ]
end
elixir_conf_eu.png
# test/test_helper.exs
Mox.defmock(Demo.Events.EventEmitterMock, for: Demo.Events.EventEmitter)

ExUnit.start()
elixir_conf_eu.png

defmodule Demo.Validators.AddressValidatorTest do
  use ExUnit.Case, async: true
  alias Demo.Validators.AddressValidator, as: Subject

  import Mox

  setup :verify_on_exit!

  test "it emits the :address_validated event" do
    Demo.Events.EventEmitterMock
    |> expect(:emit, fn "validate_address" -> :ok end)

    Subject.validate("test address")
  end
end
elixir_conf_eu.png

Two testing principles

elixir_conf_eu.png

Don't Mock What You Don't Own

elixir_conf_eu.png

dont_mock_dont_own.png

elixir_conf_eu.png

dont_mock_dont_own_1.png

elixir_conf_eu.png

dont_mock_dont_own_2.png

elixir_conf_eu.png

DAMP not DRY

elixir_conf_eu.png

describe "when the address was already validated and the feature flag is enabled" do
  setup [:address_already_validated, :enabled_feature_flag]

  test "it runs the validation again" do
    # ...
  end

end

describe "when the address was already validated and the feature flag is disabled" do
  setup [:address_already_validated, :disable_feature_flag]

  test "it does not run the validation again" do
    # ...
  end

end
elixir_conf_eu.png

The testing pyramid!

testing_pyramid_service_level.png

elixir_conf_eu.png

The testing pyramid!

testing_pyramid_think.png

elixir_conf_eu.png

microservices_arch.png

elixir_conf_eu.png

microservices_arch_calls.png

elixir_conf_eu.png

bypass.png

elixir_conf_eu.png

# mix.exs
defp deps do
  [
    # ...
    {:mox, "~> 0.5", only: :test},
    {:bypass, "~> 1.0", only: :service_level_test}
  ]
end
elixir_conf_eu.png
# service_level_test/test_helper.exs
Ecto.Adapters.SQL.Sandbox.mode(Demo.Repo, :manual)
Application.ensure_all_started(:bypass)

ExUnit.start()
elixir_conf_eu.png

defmodule Demo.ServiceLevelTest do
  use Demo.DataCase

  @address_url "localhost:4000/api/address"

  setup do
    bypass = Bypass.open()
    {:ok, bypass: bypass}
  end

  test "valid address from Service X renders a successful response", %{bypass: bypass} do
    expected_response = %{"data" => %{"address" => %{
      "street" => "Sesame St.",
      "town" => "Manhattan"
    }}}

    Bypass.expect bypass, "GET", "/service_x/api/some_endpoint", fn conn ->
      Plug.Conn.resp(conn, 200, Poison.encode!(%{status: "ok", valid: true}))
    end

    {:ok, response} =
      HTTPoison.post!(@address_url, Poison.encode!(%{street: "Sesame St.", town: "Manhattan"}))
      |> Map.get(:body)
      |> Poison.decode()

    assert response == expected_response
  end
end
elixir_conf_eu.png

Over time, the mock may (will?) diverge from the real thing

elixir_conf_eu.png

Meanwhile in production…

this_is_fine.jpg

elixir_conf_eu.png

Enter Contract Testing! 🕺

elixir_conf_eu.png

pact_elixir.png

elixir_conf_eu.png

pact.png

pact_diagram.png

elixir_conf_eu.png

pact.png

pact_diagram_first_step.png

elixir_conf_eu.png

defmodule Demo.PactElixir.ServiceX do
  alias PactElixir.PactMockServer
  import PactElixir.Dsl

  setup do
    provider = new_service_provider()
    {:ok, mock_server_pid} = start_supervised({PactMockServer, provider})
    port = PactMockServer.port(mock_server_pid)
    {:ok, mock_server_pid: mock_server_pid, port: port}
  end

  describe "Demo talks to ServiceX" do
    test "check address is valid", %{mock_server_pid: mock_server_pid, port: port} do
      expected_response = %{"status" => "ok", "valid" => true}
      path = "/api/some_endpoint"

      {:ok, response} =
        HTTPoison.get!("http://localhost:#{port}#{path}")
        |> Map.get(:body)
        |> Poison.decode()

      assert response == expected_response
      assert {:ok} == PactMockServer.write_pact_file(mock_server_pid)
    end
  end
  # ...
end
elixir_conf_eu.png
defp new_service_provider do
  %{provider: "ServiceX", consumer: "Demo"}
  |> PactElixir.Dsl.service_provider()
  |> add_interaction(
    "Check if address is valid",
    given("Valid address is available to Demo"),
    with_request(method: :get, path: "/api/some_endpoint"),
    will_respond_with(status: 200, body: %{status: "ok", valid: true})
  )
end
elixir_conf_eu.png
defmodule Demo.PactElixir.ServiceX do
  alias PactElixir.PactMockServer
  import PactElixir.Dsl

  setup do
    provider = new_service_provider()
    {:ok, mock_server_pid} = start_supervised({PactMockServer, provider})
    port = PactMockServer.port(mock_server_pid)
    {:ok, mock_server_pid: mock_server_pid, port: port}
  end

  describe "Demo talks to ServiceX" do
    test "check address is valid", %{mock_server_pid: mock_server_pid, port: port} do
      expected_response = %{"status" => "ok", "valid" => true}
      path = "/api/some_endpoint"

      {:ok, response} =
        HTTPoison.get!("http://localhost:#{port}#{path}")
        |> Map.get(:body)
        |> Poison.decode()

      assert response == expected_response
      assert {:ok} == PactMockServer.write_pact_file(mock_server_pid)
    end
  end
end
elixir_conf_eu.png

pact.png

pact_diagram_second_step.png

elixir_conf_eu.png
{
  "consumer": {
    "name": "Demo"
  },
  "provider": {
    "name": "ServiceX"
  },
  "interactions": [
    {
      "description": "Check if address is valid",
      "providerStates": [
        {"name": "Valid address is available to Demo"}
      ],
      "request": {
        "body": "",
        "headers": {},
        "method": "GET",
        "path": "/api/some_endpoint"
      },
      "response": {
        "body": {
          "status": "ok",
          "valid": true
        },
        "headers": {},
        "status": 200
      }
    // ...
elixir_conf_eu.png

pact.png

pact_diagram_final_step.png

elixir_conf_eu.png

The testing pyramid!

testing_pyramid_e2e.png

elixir_conf_eu.png

🤷‍♂️

cucumber.png

elixir_conf_eu.png

microservices_arch.png

elixir_conf_eu.png

microservices_arch_user.png

elixir_conf_eu.png

microservices_arch_user_call.png

elixir_conf_eu.png

microservices_arch_report.png

elixir_conf_eu.png

microservices_arch_report_call.png

elixir_conf_eu.png

microservices_arch_our_hl.png

elixir_conf_eu.png

For Reference:

elixir_conf_eu.png

Where does this leave us?

$ MIX_ENV=test mix test
.....................................................................
.....................................................................
.............................
Finished in 6.3 seconds
826 tests, 0 failures, 0 skipped

Randomized with seed 687577

$ MIX_ENV=service_level_test mix test
.....................................................................
.....................................................................
.......................................................
Finished in 28.2 seconds
196 tests, 0 failures

Randomized with seed 283663
elixir_conf_eu.png

Wrapping Up

  • Tests are not just about regression safety
  • Write your tests as simple and straightforward as possible
  • When creating mocks, do so based on behaviours
  • Explore different kinds of testing
  • Pick the ones right for your context
elixir_conf_eu.png

  • andre.png & daniel.png
  • mastering_elixir.png
elixir_conf_eu.png

lisbon_elixir.png

elixir_conf_eu.png

Thank you! 🙇

elixir_conf_eu.png

Questions?

caixinha.pt

gitlab.png github.png elixir_forum.png slack.png @dcaixinha

elixir_conf_eu.png