Commit 458cfe53 authored by Siebers, Michael's avatar Siebers, Michael
Browse files

Merge branch '23-system-tests-only' into '23-automate-api-tests'

System Tests

See merge request cogsys/dare2del/demonstrator!26
parents 338ee3b6 53521092
......@@ -347,8 +347,8 @@ system_test:prepare_container:
&& ($CI_OPEN_MERGE_REQUESTS == null || $CI_OPEN_MERGE_REQUESTS == "")
&& $FORCE_SYSTEM_TESTS == "yes"
system_test:
stage: build
.system_test:run:
stage: validate
image: ${CI_REGISTRY_IMAGE}/${SYSTEM_TEST_IMAGE}:${CI_COMMIT_REF_SLUG}
before_script:
- node --version
......@@ -356,8 +356,7 @@ system_test:
- swipl --version
script:
- swipl daemon.pl --user nobody
- newman run tests/api/test_suite.json -e tests/api/env.json --reporters junit --reporter-junit-export="api-report.xml" --bail
after_script:
- newman run tests/api/${TEST_NAME}-tests.json -e tests/api/env.json --reporters junit --reporter-junit-export="api-report.xml"
- swipl daemon.pl --shutdown
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
......@@ -376,6 +375,101 @@ system_test:
reports:
junit: api-report.xml
.system_test:run_with_data:
extends: .system_test:run
script:
- swipl daemon.pl --user nobody
- newman run tests/api/${TEST_NAME}-tests.json -e tests/api/env.json -d tests/api/${TEST_NAME}-data.json --reporters junit --reporter-junit-export="api-report.xml"
- swipl daemon.pl --shutdown
.system_test:run_with_data_and_db:
extends: .system_test:run
script:
- cp -f tests/api/${TEST_NAME}-theory_bg.db theory_bg.db
- chown webapi theory_bg.db
- swipl daemon.pl --user webapi
- newman run tests/api/${TEST_NAME}-tests.json -e tests/api/env.json -d tests/api/${TEST_NAME}-data.json --reporters junit --reporter-junit-export="api-report.xml"
- swipl daemon.pl --shutdown
.system_test:run_with_db:
extends: .system_test:run
script:
- cp -f tests/api/${TEST_NAME}-theory_bg.db theory_bg.db
- chown webapi theory_bg.db
- swipl daemon.pl --user webapi
- newman run tests/api/${TEST_NAME}-tests.json -e tests/api/env.json --reporters junit --reporter-junit-export="api-report.xml"
- swipl daemon.pl --shutdown
system_test:up:
extends: .system_test:run
variables:
TEST_NAME: up_and_serving
system_test:endpoint_bg@add:
extends: .system_test:run_with_data
variables:
TEST_NAME: endpoint_bg@add
system_test:endpoint_bg@remove:
extends: .system_test:run_with_data_and_db
variables:
TEST_NAME: endpoint_bg@remove
system_test:endpoint_bg@clear:
extends: .system_test:run_with_db
variables:
TEST_NAME: endpoint_bg@clear
system_test:endpoint_bg@show:
extends: .system_test:run_with_data_and_db
variables:
TEST_NAME: endpoint_bg@show
system_test:endpoint_irrelevant@file:
extends: .system_test:run_with_data_and_db
variables:
TEST_NAME: endpoint_irrelevant@file
allow_failure: true # see issue #63
system_test:endpoint_explain:
extends: .system_test:run_with_data_and_db
variables:
TEST_NAME: endpoint_explain
allow_failure: true # see issue #63
system_test:robust against wrong method:
extends: .system_test:run_with_data
variables:
TEST_NAME: robust_wrong_method
system_test:robust against wrong body:
extends: .system_test:run_with_data
variables:
TEST_NAME: robust_wrong_content_payload
allow_failure: true # see issue #61
system_test:robust against superfluous payload:
extends: .system_test:run_with_data
variables:
TEST_NAME: robust_wrong_content_empty
allow_failure: true # see issue #61
system_test:bg robust against non-item payload:
extends: .system_test:run_with_data
variables:
TEST_NAME: robust_bg_non_item_payload
system_test:irrelevant robust against non-path payload:
extends: .system_test:run_with_data
variables:
TEST_NAME: robust_irrelevant@file_non_path_payload
system_test:explain robust against non-path payload:
extends: .system_test:run_with_data
variables:
TEST_NAME: robust_explain_non_path_payload
# If no FORCE_... variable is set to "yes", no pipeline is created for merge
# requests targetting a non-default branch. Thus, the MR cannot be merged. This
......
......@@ -2,35 +2,43 @@
In this document, we will describe the general input output systematics of the WebAPI and its defined endpoints for interaction. Each category will be detailed in its own section below. For each endpoint detailed in the sections, we will give the expected HTTP request method, handled parameters, the expected body, possible HTTP return codes, the response body on success, and give an example call using the [curl command line tool](https://curl.se/).
1. [JSON Types and Objects ](<#json_types>)
*Categories*:
2. [Background Knowledge Manipulation](<#bg>)
3. [Irrelevance](<#irrelevance>)
4. [Server Information](<#server>)
5. [Documentation](<#documentation>)
In general, the request bodies (if required) and responses are formatted as JSON documents.
Thus, we will present custom JSON types and objects used throughout the endpoint descriptions first.
For limitations on JSON parsing and generation see section [Limitations](<#limitations>).
The general input and output of the API is implemented with request/response methods that entail a body formatted as JSON. Therefore we will present custom JSON types and objects used throughout the endpoint descriptions first.
## JSON Types and Objects {#json_types}
Besides the usual JSON types (string, number, object, array, boolean, and null),
Besides the usual JSON types (string, integer, number, object, array, boolean, and null),
we use the following type definitions in this documentation:
$ integer: An integer number.
$ nonnegative integer: An integer greater than or equal to 0.
$ Path: A string specifying a path. Path delimiter is the slash =|"/"|=.
A Path is interpreted as pseudo-absolute path. That means that every
path is considered absolute and not interpreted relative to some
other path.
For example the path =|"A/B"|= is interpreted as =B= in
directory =A=, where =A= itself is in no other directory. Thus, =A=
is assumed to be _|a|_(!) root directory. The empty string (=|""|=)
is not a valid Path. The *nix root is represented by a single slash
(=|"/"|=). No other Path may end in a slash!
$ Path: A pseudo-absolute file system path (see [Path](<#json_path>)).
$ File: An object representing a file on disk (see [File](<#json_file>)).
$ Directory: An object representing a directory on disk (see [Directory](<#json_directory>)).
$ Item: An object representing a file or a directory on disk.
$ Manipulation Response: A family of objects used as response in background manipulation (see [Manipulation Response](<#json_response>)).
$ Item: An object representing either a file or a directory on disk.
$ Error Response: The response body returned if the server encounters an error (see [Error Response](<#json_error>)).
### Path {#json_path}
A _Path_ is a string specifying a file system path. Path delimiter is the slash =|"/"|=.
A Path is interpreted as pseudo-absolute path. That means that every
path is considered absolute and not interpreted relative to some
other path.
For example the path =|"A/B"|= is interpreted as =B= in
directory =A=, where =A= itself is in no other directory. Thus, =A=
is assumed to be _|a|_(!) root directory. The empty string (=|""|=)
is not a valid Path. The *nix root is represented by a single slash
(=|"/"|=). No other Path may end in a slash!
A _Path_ follows [this JSON schema](<schema/type-path-schema.json>).
### File {#json_file}
A file is represented as a JSON object with the following properties:
......@@ -56,26 +64,16 @@ A directory is represented as a JSON object with the following properties:
[JSON Example for a valid directory](<schema/type-directory-example.json>), accompanying schema definition [here](<schema/type-directory-schema.json>).
### Manipulation Response {#json_response}
The API response after a completed action is represented as a JSON object with the following properties:
| Property | Type | Example | Description |
| `Action` | nonnegative integer | =21= | The number of items to which `action` was successfully applied. |
| received | nonnegative integer | =42= | The number of items successfully parsed from the client's request. |
| skipped | nonnegative integer | =0= | The number of items which were not processed. |
This type is rather a family of types, as `Action` is a variable property name. The used property depends on the action taken. An instance of this type (for `Action` =removed=) may look as follows:
### Error Response {#json_error}
When a request does not succeed or the server encounters an error it returns with a status code outside the 200 range. This usually signals an error (in the 400 or 500 range). If the requested endpoint is not serving the documentation (see section [Documentation](<#documentation>)), the response body consists of a JSON object detailing the error.
```
{
"received": 1,
"removed": 1,
"skipped": 0
}
```
For the accompanying schema definition see [here](<schema/type-manipulation-response-schema.json>).
| Property | Type | Required | Example | Description |
| code | nonnegative integer | yes | =|400|= | The status code this error caused. _Must_ be the same as the returned status code. |
| location | string | no | =|"/doc/index.html"|= | The location this error refers to. |
| message | string | yes | =|"Unexpected end of request body"|= | A detailed message what caused the error. |
| method | string | no | =|"OPTIONS"|= | The HTTP method of the offending endpoint. |
[JSON Example for an error response](<schema/response-error-example.json>), accompanying schema definition [here](<schema/response-error-schema.json>).
## Background Knowledge Manipulation {#bg}
Background Knowledge Manipulation can be done via the endpoints ``/bg/show``, ``/bg/add``, ``/bg/remove`` and ``/bg/clear`` which are described below.
......@@ -97,7 +95,7 @@ None.
| --- | --- | --- | --- |
| Ok | `200`| Request successful | a JSON object with properties `item_count` and `list` |
The property `item_count` (_nonnegative integer_) contains the number of items stored in the background knowledge. All items in the background knowledge are listed in `list` (list of [item](<#json_types>)s).
The property `item_count` (_nonnegative integer_) contains the number of items stored in the background knowledge. All items in the background knowledge are listed in `list` (list of [item](<#json_types>)s). The response body follows this [JSON schema](</doc/schema/response-bg-show-schema.json>).
#### Example Call
......@@ -140,13 +138,23 @@ curl --request GET 'http://localhost:4444/bg/show'
| --- | --- | --- | --- |
| _item_ or an array of _item_ | _see example call_ | Required | Asserts the arguments to the background knowledge.|
#### Return value
| Case | Code | Description | Response Body |
| --- | --- | --- | --- |
| Created | `201` | Response if at least one element is successfully added | a _Manipulation Response_ with action `added` |
| No Change | `200` | Response if the item(s) are already present | a _Manipulation Response_ with action `added` |
| Created | `201` | Response if at least one element is successfully added | see below |
| No Change | `200` | Response if the item(s) are already present | see below |
In both cases, the response body is a JSON object with the following properties:
The API response after a completed action is a JSON object with the following properties:
| Property | Type | Example | Description |
| added | nonnegative integer | =21= | The number of items successfully added. |
| received | nonnegative integer | =42= | The number of items successfully parsed from the request body. |
| skipped | nonnegative integer | =0= | The number of items which could not be added (since they were already present). |
The result body is defined in this [JSON schema](<schema/response-bg-add-schema.json>), an example can be found below or in [the example file](<schema/response-bg-add-example.json>).
#### Example Call
......@@ -189,12 +197,16 @@ In contrast to `bg/clear`, remove allows for retracting the provided items only.
| Case | Code | Description | Response Body |
| --- | --- | --- | --- |
| Ok | `200` | Success | a _Manipulation Response_ with action `removed` |
| Ok | `200` | Success | see below |
The API response after a completed action is a JSON object with the following properties:
#### Notes
| Property | Type | Example | Description |
| removed | nonnegative integer | =21= | The number of items successfully removed. |
| received | nonnegative integer | =42= | The number of items successfully parsed from the request body. |
| skipped | nonnegative integer | =0= | The number of items which could not be removed (since they were not present). |
* See [JSON type definition](<#json_types>) for the request and reply json schemas.
* Will respond with `200` also if the item(s) were not present (i.e. skipped == received)
The result body is defined in this [JSON schema](<schema/response-bg-remove-schema.json>). , an example can be found below or in [the example file](<schema/response-bg-remove-example.json>).
#### Example Call
......@@ -242,6 +254,8 @@ None.
- `removed` is the number of removed items
- `skipped` is the number of ot removed items. Should always be zero.
The response body follows this [JSON schema](</doc/schema/response-bg-clear-schema.json>).
#### Example Call
```bash
......@@ -281,6 +295,7 @@ Queries whether a _single file_ is irrelevant.
| --- | --- | --- | --- |
| Ok | `200` | Success | JSON object with property `result` (=boolean=) |
The response body follows this [JSON schema](</doc/schema/response-irrelevant-file-schema.json>).
#### Example Call
......@@ -321,16 +336,21 @@ Explains why an irrelevant file is irrelevant.
| Case | Code | Description | Response Body |
| --- | --- | --- | --- |
| Ok | `200` | explanation(s) successfully created | JSON object with properties `explanations` and `further_explanations` (=boolean=) |
| Bad Request | `400` | item is not irrelevant and thus requesting an explanation is not possible | JSON object with properties `code` and `message` |
| Bad Request | `400` | item is not irrelevant and thus requesting an explanation is not possible | an [Error response](<#json_error>) |
- `explanations`: A list of explanations (see below).
- `further_explanations`: `true` if more explanations are available
- `code`: the HTTP status code (400)
- `message`: an error message
#### Explanation type
On success, the response body follows this [JSON schema](</doc/schema/response-explain-schema.json>). An example for a response body can be found below and in the [example file](</doc/schema/response-explain-example.json>).
TODO
#### Explanations
An explanation consists of four parts:
- the path of the irrelevant file explained (`abs_path`, Path),
- a top-level view reasoning why the file is irrelevant (`reasoning`),
- a detailed step-by-step reasoning why the file is irrelevant (`reasoning_details`), and
- references to the 'real-world' file system (`references`).
#### Example Call
......@@ -429,8 +449,9 @@ status of the server.
| Property | Type | Description |
| --- | --- | --- |
| =status= | string | The status of the server. Currently, always =|"running"|=. |
| =version= | string | The version number of the WebAPI server. |
| =version= | string | The [semantic version number](https://semver.org) of the WebAPI. |
The response body follows this [JSON schema](</doc/schema/response-state-schema.json>).
#### Example Call
......@@ -447,6 +468,22 @@ curl --request GET 'http://localhost:4444/state'
}
```
## Limitations {#limitations}
Due to technical difficulties, neither is perfect. The library-standard HTTP server and the included JSON parsing mechanism are not perfect. Thus, some limitations on parsing JSON requests and generating JSON responses are imposed.
### Request Bodies with JSON
The current implementation restricts the JSON content of request bodies. Only the first JSON _entity_ is parsed, the remaining text in the body is *silently* discarded. The parsed entity may be of any basic JSON type, like a string, an object, or an array. This may lead to unexpected results on erroneous input. For example, the incomplete array =|1,2,3]|= would be parsed as the integer `1` while the remainig characters are ignored without warning.
However, this limitation only affects the body as a whole. For instance, the body =|{"key": 1,2,3]}|= results in a parsing error since JSON properties must be strings (and `2` is not).
### JSON Arrays
Unlike the JSON standard, the JSON parser ignores a trailing comma in arrays.
### JSON Error Bodies
The values of some HTTP headers are inspected by the HTTP server before beeing handed over to the WebAPI module. If the value for the header field is not permissible, the HTTP server responds with a `400 Bad Request` response. However, the body of the response will not be an JSON document but an HTML document.
> For a technical discussion of the underlying problem see [issue #62](https://gitlab.rz.uni-bamberg.de/cogsys/dare2del/demonstrator/-/issues/62)
## Documentation {#documentation}
When started with the =|--documentation|= switch, the WebAPI server serves its own documentation as HTML pages. The documentation can be accessed using =GET= requests, for example using a conventional web browser.
......
......@@ -31,7 +31,7 @@ The Reasoning WebAPI server provides its own documentation. It is split in three
### User documentation
1. [User Guide](<user_guide.html>) (incomplete)
1. [API endpoints](<api_endpoints.html>)
1. [JSON Schemas](<schemas.html>)
1. [JSON Schemas](<schema/index.html>)
### Admin documentation
1. [Admin Guide](<admin_guide.html>) (incomplete)
......
{
"abs_path": "KI Konferenzen/Berlin2018/Folien_v3.pptx",
"references": [
{
"id": 0,
"referenced_entities": [
{
"abs_path": "KI Konferenzen/Berlin2018/Folien_v3.pptx",
"property": "name"
}
]
},
{
"id": 1,
"referenced_entities": [
{
"abs_path": "KI Konferenzen/Berlin2018/Folien_final.pptx",
"property": "name"
}
]
},
{
"id": 2,
"referenced_entities": [
{
"abs_path": "KI Konferenzen/Berlin2018/Folien_v3.pptx",
"property": "mtime"
},
{
"abs_path": "KI Konferenzen/Berlin2018/Folien_final.pptx",
"property": "mtime"
}
]
}
],
"reasoning": [
{
"text": "die Datei "
},
{
"text": "Folien_v3.pptx",
"ref_id": 0
},
{
"text": " ist eine Vorversion von "
},
{
"text": "Folien_final.pptx",
"ref_id": 1
}
],
"reasoning_details": [
[
{
"text": "beref_ide Dateien sind im selben Verzeichnis"
}
],
[
{
"text": "Folien_v3.pptx",
"ref_id": 0
},
{
"text": " und "
},
{
"text": "Folien_final.pptx",
"ref_id": 1
},
{
"text": "haben fast den gleichen Inhalt"
}
],
[
{
"text": "die Dateinamen "
},
{
"text": "Folien_v3.pptx",
"ref_id": 0
},
{
"text": " und "
},
{
"text": "Folien_final.pptx",
"ref_id": 1
},
{
"text": " beginnen mit (mindestens) 5 gleichen Buchstaben"
}
],
[
{
"text": "Datei "
},
{
"text": "Folien_final.pptx",
"ref_id": 1
},
{
"text": " ist "
},
{
"text": "neuer",
"ref_id": 2
},
{
"text": " als Datei "
},
{
"text": "Folien_v3.pptx",
"ref_id": 0
}
]
]
}
{
"definitions": {},
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://example.com/object1603988523.json",
"title": "Root",
"type": "object",
"description": "Response schema for an object that is classified as irrelevant, including explanation messages and details.",
"properties": {
"abs_path": {
"$id": "#root/abs_path",
"title": "Abs_path",
"description": "The object classified as irrelevant.",
"type": "string",
"default": ""
},
"references": {
"$id": "#root/references",
"title": "References",
"description": "A list of objects that are used for eplaining the (ir)relevance",
"type": "array",
"default": [],
"items":{
"$id": "#root/references/items",
"title": "Items",
"type": "object",
"properties": {
"id": {
"$id": "#root/references/items/id",
"title": "Id",
"type": "integer",
"default": 0
},
"referenced_entities": {
"$id": "#root/references/items/referenced_entities",
"title": "Referenced_entities",
"type": "array",
"default": [],
"items":{
"$id": "#root/references/items/referenced_entities/items",
"title": "Items",
"type": "object",
"properties": {
"abs_path": {
"$id": "#root/references/items/referenced_entities/items/abs_path",
"title": "Abs_path",
"type": "string",
"default": ""
},
"property": {
"$id": "#root/references/items/referenced_entities/items/property",
"title": "Property",
"type": "string",
"default": ""
}
}
}
}
}
}
},
"reasoning": {
"$id": "#root/reasoning",
"title": "Reasoning",
"description": "A short description why the object is irrelevant.",
"type": "array",
"default": [],
"items":{
"$id": "#root/reasoning/items",
"title": "Items",
"type": "object",
"properties": {
"text": {
"$id": "#root/reasoning/items/text",
"title": "Text",
"type": "string",
"default": ""
}
}
}
},
"reasoning_details": {
"$id": "#root/reasoning_details",
"title": "Reasoning_details",
"description": "A in depth list.",
"type": "array",
"default": [],
"items":{
"$id": "#root/reasoning_details/items",
"title": "Items",
"type": "array",
"default": [],
"items":{
"$id": "#root/reasoning_details/items/items",
"title": "Items",
"type": "object",
"properties": {
"text": {
"$id": "#root/reasoning_details/items/items/text",
"title": "Text",
"type": "string",
"default": ""
}
}
}
}
}
}
}
[
{
"type" : "file",
"abs_path": "/A/B/Filename.exe",
"file_size": 1024,
"creation_time": 1604235569,
"modification_time": 1604235569,
"access_time": 1604235569,
"change_time": 1604235569,
"media_type": "img",
"filename_extension": "png"
},{
"type" : "file",
"abs_path": "/A/B/Filename2.exe",
"file_size": 1024,
"creation_time": 1604235569,
"modification_time": 1604235570,
"access_time": 1604235569,
"change_time": 1604235569,
"media_type": "img",
"filename_extension": "png"
},
{
"type" : "directory",
"abs_path": "/A/B",
"creation_time": 1604235509
}
]
\ No newline at end of file
# JSON Schemas
API POST requests and responses contain payload formatted as JSON. For each respective request and response, the schemas are linked in the [API endpoint documentation](</doc/api_endpoints.html>). This page provides an overview of the schemas.
Most API endpoint require a JSON document as requests body and respond with a JSON document. For each respective request and response, the schemas are linked in the [API endpoint documentation](</doc/api_endpoints.html>). This page provides an overview.
## Types
| Type | Schema | Example |
......@@ -7,4 +7,17 @@ API POST requests and responses contain payload formatted as JSON. For each resp
| =Directory= | [Schema](<type-directory-schema.json>) | [Example](<type-directory-example.json>) |