test: add generated BDD test runner

DEVELOPMENT.md

@@ -28,7 +28,19 @@ This project contains BDD tests.
 __Never__ run the test suite against an organization with production data.
 
 Tests can be executed via [`cucumber`](https://cucumber.io/docs/guides/).
-You can optionally add `features/v<NUMBER>/<FILENAME>.feature:<LINE_NUMBER>` to filter individual tests.
+
+Generated SDKs include the language-neutral request plans and replay server
+under `generated-test`. The normal test command starts and stops the replay
+server through Cucumber's suite hooks, without an API spec checkout or
+generator tooling:
+
+```shell
+./run-tests.sh
+```
+
+The Ruby Cucumber runner still invokes the generated SDK method and validates
+its response. Request bodies and parameters come from the generated scenario
+plan, while recorded HTTP interactions are replayed by the shared server.
 
 By default integration tests use recorded API responses stored in cassettes. To record new API responses run the tests with `RECORD=true`. To run integration tests against API without recording cassettes, run the tests with `RECORD=none`.
 
@@ -39,4 +51,4 @@ generated code, only commit test files being updated and any updated cassettes.
 ### CI
 
 In rare ocassions, the test suite may fail due to a corrupted cache in `ruby/setup-ruby` action.
-Please update `CACHE_VERSION` secret in https://github.com/DataDog/datadog-api-client-ruby/settings/secrets/actions/CACHE_VERSION with a new unique value.
+Please update `CACHE_VERSION` secret in https://github.com/DataDog/datadog-api-client-ruby/settings/secrets/actions/CACHE_VERSION with a new unique value.

features/step_definitions/request.rb

@@ -19,7 +19,14 @@ def configuration
 
   def from_env(configuration)
     configuration.configure do |c|
-      if ENV.key? 'DD_TEST_SITE' then
+      if ENV.key? 'DD_TEST_SERVER_URL' then
+        uri = URI(ENV['DD_TEST_SERVER_URL'])
+        name = uri.host
+        name += ":#{uri.port}" unless [80, 443].include?(uri.port)
+        c.server_index = 1
+        c.server_variables[:name] = name
+        c.server_variables[:protocol] = uri.scheme
+      elsif ENV.key? 'DD_TEST_SITE' then
         c.server_index = 2
         c.server_variables[:site] = ENV['DD_TEST_SITE']
       end
@@ -34,7 +41,7 @@ def from_env(configuration)
   end
 
   def api_client
-    @api_client ||= api::APIClient.new configuration
+    @api_client ||= add_test_server_session(api::APIClient.new(configuration))
   end
 
   def api_error
@@ -48,7 +55,7 @@ def sleep_after_request
   def unique
     now = Time.now.utc
     scenario_name = @scenario.name.gsub(/[^A-Za-z0-9]+/, '_')[0, 100]
-    prefix = ENV["RECORD"] == "none" ? "Test-Ruby" : "Test"
+    prefix = ENV["RECORD"] == "none" && !test_server_enabled? ? "Test-Ruby" : "Test"
     @unique ||= "#{prefix}-#{scenario_name}-#{now.to_i}"
   end
 
@@ -113,10 +120,14 @@ def path_parameters
     @path_parameters ||= {}
   end
 
+  def test_features_root
+    File.expand_path('..', __dir__)
+  end
+
   def undo_operations
     return @undo_operations if @undo_operations
     @undo_operations = {}
-    Dir.glob(File.join(__dir__, '..', "v*", 'undo.json')).each do |undo_path|
+    Dir.glob(File.join(test_features_root, "v*", 'undo.json')).each do |undo_path|
       m = File.expand_path(undo_path).match /features\/v(?<version>\d+)\/.*/
       version = m[:version]
       @undo_operations[version] = {}
@@ -153,6 +164,7 @@ def build_undo_for(version, operation_id, api_instance = nil)
       undo_configuration.api_key = ENV["DD_TEST_CLIENT_API_KEY"]
       undo_configuration.application_key = ENV["DD_TEST_CLIENT_APP_KEY"]
       undo_api_client = undo_api::APIClient.new undo_configuration
+      add_test_server_session(undo_api_client)
       api_instance = undo_api.const_get("V#{version}").const_get(api_name).new undo_api_client
     end
 
@@ -216,6 +228,7 @@ def build_given(api_version, operation)
     given_configuration.api_key = ENV["DD_TEST_CLIENT_API_KEY"]
     given_configuration.application_key = ENV["DD_TEST_CLIENT_APP_KEY"]
     given_api_client = given_api::APIClient.new given_configuration
+    add_test_server_session(given_api_client)
     given_api_instance = given_api.const_get("V#{api_version}").const_get(api_name).new given_api_client
     method = given_api_instance.method("#{operation_name}_with_http_info".to_sym)
 
@@ -233,7 +246,7 @@ def build_given(api_version, operation)
       result = fixtures.lookup(p["source"]) if p.key? "source"
       model = ScenariosModelMappings["v#{api_version}.#{operation["operationId"]}"][p["name"]]
       if model == 'File'
-        result = {p["name"].to_sym => File.open(File.join(__dir__, "..", "v" + api_version, result))}
+        result = {p["name"].to_sym => File.open(File.join(test_features_root, "v" + api_version, result))}
       end
       result
     end if operation["parameters"]
@@ -274,10 +287,14 @@ def build_given(api_version, operation)
     result
   end
 
-  def model_builder(param, obj)
-    model = ScenariosModelMappings["v#{@api_version}.#{@operation_id}"][param]
+  def model_builder(param, obj, schema = nil)
+    model = if schema
+      test_runner_model(schema)
+    else
+      ScenariosModelMappings["v#{@api_version}.#{@operation_id}"][param]
+    end
     if model == 'File'
-      return File.open(File.join(__dir__, "..", "v" + @api_version, obj))
+      return File.open(File.join(test_features_root, "v" + @api_version, obj))
     end
     @api_client.convert_to_type(obj, model, "V#{@api_version}")
   end
@@ -305,17 +322,23 @@ def model_builder(param, obj)
 end
 
 Given(/^body with value (.*)$/) do |body|
+  next if test_runner_enabled?
+
   body_hash = JSON.parse(body.templated(fixtures), {:symbolize_names => true})
   opts[:body] = model_builder("body", body_hash)
 end
 
 Given(/^body from file "(.*)"$/) do |file|
-  body = File.read(File.join(__dir__, "..", "v" + @api_version, file))
+  next if test_runner_enabled?
+
+  body = File.read(File.join(test_features_root, "v" + @api_version, file))
   body_hash = JSON.parse(body.templated(fixtures), {:symbolize_names => true})
   opts[:body] = model_builder("body", body_hash)
 end
 
 Given(/^request contains "([^"]+)" parameter from "([^"]+)"$/) do |parameter_name, fixture_path|
+  next if test_runner_enabled?
+
   param_value = model_builder(parameter_name.to_parameter, fixtures.lookup(fixture_path))
   param_key = parameter_name.to_parameter.to_sym
   opts[param_key] = param_value
@@ -326,6 +349,8 @@ def model_builder(param, obj)
 end
 
 Given(/^request contains "([^"]+)" parameter with value (.+)$/) do |parameter_name, value|
+  next if test_runner_enabled?
+
   param_value = model_builder(parameter_name.to_parameter, JSON.parse(value.templated fixtures))
   param_key = parameter_name.to_parameter.to_sym
   opts[param_key] = param_value
@@ -336,17 +361,22 @@ def model_builder(param, obj)
 end
 
 Given(/^new "([^"]+)" request$/) do |name|
+  next if test_runner_enabled?
+
   @operation_id = name
   @api_method = @api_instance.method("#{name.snakecase}_with_http_info".to_sym)
 end
 
 When('the request is sent') do
+  prepare_test_runner_request if test_runner_enabled?
   params = @api_method.parameters.select { |p| p[0] == :req }.map { |p| opts.delete(p[1]) }
   undo_builder = build_undo_for(@api_version, @api_method.name.to_s.chomp('_with_http_info'))  # fail early on missing undo method
 
   begin
     @response = @api_method.call(*params, opts)
   rescue api_error => e
+    raise e if e.response_headers && e.response_headers['x-openapi-test-error']
+
     # If we have an exception, make a stub response object to use for assertions
     # Instead of finding the response class of the method, we use the fact that all
     # responses returned have the `1` element set to the response code
@@ -364,6 +394,7 @@ def model_builder(param, obj)
 end
 
 When('the request with pagination is sent') do
+  prepare_test_runner_request if test_runner_enabled?
   method_name = @api_method.name.to_s.chomp('_with_http_info') + "_with_pagination"
   method = @api_instance.method(method_name.to_sym)
   params = method.parameters.select { |p| p[0] == :req }.map { |p| opts.delete(p[1]) }
@@ -373,6 +404,8 @@ def model_builder(param, obj)
     method.call(*params, opts) { |item| result.append(item) }
     @response = [result, 200, nil]
   rescue api_error => e
+    raise e if e.response_headers && e.response_headers['x-openapi-test-error']
+
     # If we have an exception, make a stub response object to use for assertions
     # Instead of finding the response class of the method, we use the fact that all
     # responses returned have the `1` element set to the response code
@@ -433,7 +466,8 @@ def model_builder(param, obj)
   expect(body.lookup response_path).to include(JSON.parse(value.templated(fixtures), :symbolize_names => true))
 end
 
-Dir.glob(File.join(__dir__, '..', "v*", 'given.json')).each do |f|
+features_root = File.expand_path('..', __dir__)
+Dir.glob(File.join(features_root, "v*", 'given.json')).each do |f|
   m = File.expand_path(f).match /features\/v(?<version>\d+)\/.*/
   version = m[:version]
   JSON.parse(File.read(f)).map do |settings|

features/support/hooks.rb

@@ -1,4 +1,14 @@
 require 'json'
+require_relative 'test_runner'
+
+BeforeAll do
+  GeneratedTestServer.start
+  puts GENERATED_TEST_RUNNER_BANNER if GeneratedTestServer.enabled?
+end
+
+AfterAll do
+  GeneratedTestServer.stop
+end
 
 Around do |scenario, block|
   current_span = Datadog::Tracing.active_span
@@ -42,22 +52,33 @@
 end
 
 Around do |scenario, block|
-  VCR.use_cassette(scenario.location.file.chomp('.feature') + "/" + scenario.name.gsub(/[^A-Za-z0-9]+/, '-')) do |cassette|
-    if !File.exist?(cassette.file) && ENV.fetch("RECORD", "false") == "false" && !scenario.match_tags?("@integration-only")
-      raise Exception.new "Cassette '#{cassette.file}' not found: create one setting `RECORD=true` or ignore it using `RECORD=none`"
+  if test_server_enabled?
+    freeze = start_test_server_session
+    begin
+      Timecop.freeze(freeze) do
+        block.call
+      end
+    ensure
+      stop_test_server_session
     end
-    File.delete(cassette.file) if ENV["RECORD"] == "true" && File.exist?(cassette.file) && !scenario.match_tags?("@replay-only")
+  else
+    VCR.use_cassette(scenario.location.file.chomp('.feature') + "/" + scenario.name.gsub(/[^A-Za-z0-9]+/, '-')) do |cassette|
+      if !File.exist?(cassette.file) && ENV.fetch("RECORD", "false") == "false" && !scenario.match_tags?("@integration-only")
+        raise Exception.new "Cassette '#{cassette.file}' not found: create one setting `RECORD=true` or ignore it using `RECORD=none`"
+      end
+      File.delete(cassette.file) if ENV["RECORD"] == "true" && File.exist?(cassette.file) && !scenario.match_tags?("@replay-only")
 
-    if use_real_time? then
-      freeze = Time.now.utc
-    else
-      File.open(cassette.file.gsub(/\.yml$/, '.frozen'), 'r') do |f|
-        freeze = Time.parse(f.readline.chomp)
+      if use_real_time? then
+        freeze = Time.now.utc
+      else
+        File.open(cassette.file.gsub(/\.yml$/, '.frozen'), 'r') do |f|
+          freeze = Time.parse(f.readline.chomp)
+        end
       end
-    end
 
-    Timecop.freeze(freeze) do
-      block.call
+      Timecop.freeze(freeze) do
+        block.call
+      end
     end
   end
 end