diff --git a/docs/_data/navigation.json b/docs/_data/navigation.json index 549c0c0b..d1ebc5c0 100644 --- a/docs/_data/navigation.json +++ b/docs/_data/navigation.json @@ -16,7 +16,7 @@ "content": [ { - "name": "Repository", + "name": "Developer Documentation", "type": "subcategory", "content": [ { @@ -26,15 +26,21 @@ "type": "page" }, { - "name": "Architecture", + "name": "ReFlex Library (.NET)", "path": "/software/repo/", - "file": "architecture.html", + "file": "library.html", "type": "page" }, { - "name": "ReFlex Library (.NET)", + "name": "Requirements Analysis", "path": "/software/repo/", - "file": "library.html", + "file": "requirements.html", + "type": "page" + }, + { + "name": "Architecture", + "path": "/software/repo/", + "file": "architecture.html", "type": "page" }, { @@ -49,6 +55,12 @@ "file": "touch-detection.html", "type": "page" }, + { + "name": "Supported Sensors", + "path": "/software/repo/", + "file": "sensors.html", + "type": "page" + }, { "name": "Interactions Format", "path": "/software/repo/", @@ -56,67 +68,34 @@ "type": "page" }, { - "name": "Supported Sensors", + "name": "Server Configuration", "path": "/software/repo/", - "file": "sensors.html", + "file": "config-file.html", "type": "page" }, { - "name": "github CI", + "name": "REST API", + "path": "/software/apps/server/", + "file": "rest-api.html", + "type": "page" + }, + { + "name": "SignalR", + "path": "/software/apps/server/", + "file": "signal-r.html", + "type": "page" + }, + { + "name": "TUIO", "path": "/software/repo/", - "file": "ci.html", + "file": "tuio.html", "type": "page" }, { - "name": "Development Documentation", + "name": "github CI", "path": "/software/repo/", - "file": "dev.html", - "type": "page", - "content": - [ - { - "name": "Requirements Analysis", - "path": "/software/repo/", - "file": "requirements.html", - "type": "page" - }, - { - "name": "Sensor Abstraction Layer", - "path": "/software/repo/", - "file": "abstraction-layer.html", - "type": "page" - }, - { - "name": "Interaction Observer", - "path": "/software/repo/", - "file": "interaction-observer.html", - "type": "page" - }, - { - "name": "Configuration File", - "path": "/software/repo/", - "file": "config-file.html", - "type": "page" - }, - { - "name": "REST API", - "path": "/software/apps/server/", - "file": "rest-api.html", - "type": "page" - }, - { - "name": "SignalR", - "path": "/software/apps/server/", - "file": "signal-r.html", - "type": "page" - }, - { - "name": "TUIO", - "path": "/software/repo/", - "file": "tuio.html", - "type": "page" - } - ] + "file": "ci.html", + "type": "page" }, { "name": "Version History", diff --git a/docs/assets/img/overview/requirement_analysis.png b/docs/assets/img/overview/requirement_analysis.png new file mode 100644 index 00000000..eed385cc Binary files /dev/null and b/docs/assets/img/overview/requirement_analysis.png differ diff --git a/docs/software/apps/server/rest-api.md b/docs/software/apps/server/rest-api.md index 35755e18..f7f7d7c5 100644 --- a/docs/software/apps/server/rest-api.md +++ b/docs/software/apps/server/rest-api.md @@ -146,6 +146,11 @@ __[⬆ back to top](#table-of-contents)__ ## Calibration +- Retrieval of current calibration data +- Specification of parameters +- Updating of calibration points +- Explicit transformation from world to image coordinates + | Method | Route | Request | Response | Description | |---|---|---|---|---| | `GET` | `/api/Calibration/FrameSize` | - | `FrameSizeDefinition` | Returns the current calibration window/frame size. | @@ -164,6 +169,9 @@ __[⬆ back to top](#table-of-contents)__ ## Tracking +- Start/stop the sensor (including emulator/recorder) +- Specifying parameters (sensor, resolution) + | Method | Route | Request | Response | Description | |---|---|---|---|---| | `GET` | `/api/Tracking` | - | `IDepthCamera[]` | Returns all available cameras. | @@ -191,6 +199,9 @@ __[⬆ back to top](#table-of-contents)__ ## Settings +- Access to the central settings of the filter pipeline +- Loading/saving the configuration + | Method | Route | Request | Response | Description | |---|---|---|---|---| | `GET` | `/api/Settings` | - | `TrackingServerAppSettings` | Returns the complete current server configuration. | @@ -225,6 +236,9 @@ __[⬆ back to top](#table-of-contents)__ ## Processing +- Start/stop the processing component +- Specifying parameters (type of processing, update rate) + | Method | Route | Request | Response | Description | |---|---|---|---|---| | `GET` | `/api/Processing/GetInterval` | - | `int` | Returns the current update interval in milliseconds. | @@ -241,6 +255,9 @@ __[⬆ back to top](#table-of-contents)__ ## Network +- Start/stop the transmission component +- Specify parameters (address, protocol, update rate) + | Method | Route | Request | Response | Description | |---|---|---|---|---| | `GET` | `/api/Network/Status` | - | `NetworkAttributes` | Returns a summarized networking status including address, port, endpoint, and interfaces. | @@ -262,6 +279,9 @@ __[⬆ back to top](#table-of-contents)__ ## Tuio +- Start/stop TUIO transmission +- Specifying parameters (version, encoding, protocol) + | Method | Route | Request | Response | Description | |---|---|---|---|---| | `GET` | `/api/Tuio/IsBroadcasting` | - | `JsonSimpleValue` | Returns whether TUIO broadcasting is active. Name: `IsBroadcasting`. | @@ -281,6 +301,9 @@ __[⬆ back to top](#table-of-contents)__ ## DepthImage +- Retrieving the point cloud +- Retrieving the vector field + | Method | Route | Request | Response | Description | |---|---|---|---|---| | `GET` | `/api/DepthImage/PointCloud` | - | `Point3[]` | Returns the current point cloud or an empty array. | @@ -290,6 +313,9 @@ __[⬆ back to top](#table-of-contents)__ ## Log +- Retrieving the log (in full / in sections) +- Sending errors for inclusion in the server log + | Method | Route | Request | Response | Description | |---|---|---|---|---| | `GET` | `/api/Log` | - | `LogMessageDetail[]` | Returns the currently buffered log messages. | @@ -302,6 +328,8 @@ __[⬆ back to top](#table-of-contents)__ ## VersionInfo +- provide server version and current dependencies + | Method | Route | Request | Response | Description | |---|---|---|---|---| | `GET` | `/api/VersionInfo` | - | `AppVersionInfo[]` | Returns the known version information for the application. | @@ -310,6 +338,8 @@ __[⬆ back to top](#table-of-contents)__ ## RecordRawDepth +- Saving the point cloud to a JSON file (surveying) + | Method | Route | Request | Response | Description | |---|---|---|---|---| | `GET` | `/api/RecordRawDepth/IsCapturing` | - | `bool` | Returns whether raw data is currently being captured. | diff --git a/docs/software/repo/config-file.md b/docs/software/repo/config-file.md index 8c2eb117..376a693c 100644 --- a/docs/software/repo/config-file.md +++ b/docs/software/repo/config-file.md @@ -1,5 +1,246 @@ --- -title: Configuration File +title: Server Configuration --- # {{ page.title }} + + +## Table of Contents + +1. [Introduction](#introduction) +2. [FilterSettingValues](#filtersettingvalues) +3. [CalibrationValues](#calibrationvalues) +4. [FrameSize](#framesize) +5. [Camera and Startup](#camera-and-startup) +6. [CameraConfigurationValues](#cameraconfigurationvalues) +7. [NetworkSettingValues](#networksettingvalues) +8. [PointCloudSettingValues](#pointcloudsettingvalues) +9. [ProcessingSettingValues](#processingsettingvalues) +10. [RemoteProcessingServiceSettingsValues](#remoteprocessingservicesettingsvalues) +11. [TuioSettingValues](#tuiosettingvalues) + +## Introduction + +`tools/ReFlex.TrackingServer/wwwroot/Config/TrackingSettings.json` stores the tracking server configuration represented by `TrackingServerAppSettings`. The sections below follow the file's structure and use its exact property names. Enum values are stored as numbers. + +**Example Value** refers to the values currently checked into the repository, not universal defaults or recommended values for another installation. Camera settings, calibration and the filter mask depend on the physical setup. Explicit C# class defaults are noted separately where relevant. Types refer to the C# model; JSON uses numbers, booleans, strings, arrays and objects. + +The [Settings REST endpoints]({{ site.baseurl }}/software/apps/server/rest-api.html#settings) expose the current configuration and operations for applying, saving, restoring and resetting it. The processing stages are described in the [pipeline documentation]({{ site.baseurl }}/software/repo/pipeline.html). + +__[⬆ back to top](#table-of-contents)__ + +## FilterSettingValues + +Filtering and interaction detection parameters (`FilterSettings`). Nested parameter groups are documented below. + +| Property | Type (.NET) | Example value | Description | +| --- | --- | --- | --- | +| `Threshold` | `float` | `156.0` | Maximum depth difference between neighbouring points before the threshold filter treats a point as an outlier; uses the depth data units. | +| `MinDistanceFromSensor` | `float` | `0.0` | Minimum accepted sensor distance. | +| `LimitationFilterType` | `LimitationFilterType` | `1` | `0`: LimitationFilter; `1`: AdvancedLimitationFilter. | +| `FilterMask` | `bool[][]?` | `Array` | Persisted advanced limitation mask, indexed as [x][y]. True marks positions accepted near the zero plane during initialization. Dimensions must match the sensor data. | +| `UseOptimizedBoxFilter` | `bool` | `true` | Selects the optimized box filter. | +| `AdvancedLimitationFilterThreshold` | `float` | `0.06` | Maximum deviation from the zero plane when initializing the mask, in depth data units. | +| `AdvancedLimitationFilterSamples` | `int` | `8` | Number of frames sampled to initialize the mask. | +| `IsLimitationFilterEnabled` | `bool` | `true` | Enables spatial limitation filtering. | +| `IsValueFilterEnabled` | `bool` | `true` | Enables invalid-value filtering. | +| `IsThresholdFilterEnabled` | `bool` | `true` | Enables depth outlier filtering. | +| `IsBoxFilterEnabled` | `bool` | `true` | Enables spatial box smoothing. | +| `MeasurePerformance` | `bool` | `true` | Enables performance measurement. | +| `BoxFilterRadius` | `int` | `4` | Spatial smoothing radius in sensor pixels. | +| `BoxFilterNumPasses` | `int` | `2` | Number of optimized box-filter passes. | +| `BoxFilterNumThreads` | `int` | `0` | Thread setting for the optimized box filter. | +| `MinAngle` | `float` | `0.5` | Angle threshold used by multi-touch extremum detection. | + +__[⬆ back to top](#table-of-contents)__ + +### BorderValue + +Sensor-space boundaries of the region processed by the limitation filter. + +| Property | Type (.NET) | Example value | Description | +| --- | --- | --- | --- | +| `Left` | `int` | `12` | Left boundary (pixels). | +| `Right` | `int` | `616` | Right boundary (pixels). | +| `Top` | `int` | `132` | Upper boundary (pixels). | +| `Bottom` | `int` | `480` | Lower boundary (pixels). | + +__[⬆ back to top](#table-of-contents)__ + +### DistanceValue + +Depth normalization and separation of detected interactions. + +| Property | Type (.NET) | Example value | Description | +| --- | --- | --- | --- | +| `Min` | `float` | `0.04` | Minimum deformation distance from the zero plane. | +| `Max` | `float` | `0.24` | Maximum deformation distance used for depth normalization. | +| `Default` | `float` | `0.77` | Sensor distance of the undeformed surface (zero plane); also the box-filter default value. | +| `InputDistance` | `float` | `40.0` | Clustering threshold compared directly with squared 2D distance between interactions, not a linear distance. | + +__[⬆ back to top](#table-of-contents)__ + +### Confidence + +Temporal stability thresholds for tracked touches. + +| Property | Type (.NET) | Example value | Description | +| --- | --- | --- | --- | +| `Min` | `int` | `2` | Minimum confidence threshold. | +| `Max` | `int` | `20` | Maximum confidence accumulated for a tracked touch. | + +__[⬆ back to top](#table-of-contents)__ + +### SmoothingValues + +Interaction history, matching and temporal smoothing (`SmoothingParameter`). + +| Property | Type (.NET) | Example value | Description | +| --- | --- | --- | --- | +| `InteractionHistorySize` | `int` | `15` | Number of frames retained in the interaction history. | +| `NumSmoothingSamples` | `int` | `9` | Number of samples used for temporal smoothing. | +| `MaxNumEmptyFramesBetween` | `int` | `3` | Allowed gap of empty frames when matching touches. | +| `TouchMergeDistance2D` | `float` | `72.0` | Distance threshold for matching touches across frames. | +| `DepthScale` | `float` | `490.0` | Scale factor for depth in interaction smoothing/matching. | +| `Type` | `FilterType` | `2` | `0`: None; `1`: MovingAverage; `2`: WeightedMovingAverage; `3`: SavitzkyGolay; `4`: PolynomialFit; `5`: WeightedPolynomialFit; `6`: Butterworth. | + +__[⬆ back to top](#table-of-contents)__ + +### ExtremumSettings + +Sampling parameters for classifying local extrema. + +| Property | Type (.NET) | Example value | Description | +| --- | --- | --- | --- | +| `NumSamples` | `int` | `10` | Number of surrounding samples. | +| `CheckRadius` | `int` | `20` | Sampling radius around the extremum. | +| `FitPercentage` | `float` | `0.9` | Required fraction of samples supporting the classification, in [0, 1]. | +| `CheckMethod` | `ExtremumTypeCheckMethod` | `1` | `0`: Global; `1`: FixedRadius; `2`: StochasticStatic; `3`: StochasticDynamic. Static random samples are reused; dynamic samples are regenerated. | + +__[⬆ back to top](#table-of-contents)__ + +## CalibrationValues + +Persisted calibration correspondences. Despite their names, the calibrator stores predefined screen positions in `SourceValues` and measured sensor positions in `TargetValues`. Three point pairs are used to compute the transformation. + +| Property | Type (.NET) | Example value | Description | +| --- | --- | --- | --- | +| `SourceValues` | `List` | `[[1200, 360, -1], [1200, 720, -1], [600, 720, -1]]` | Reference screen points as [x, y, id]; generated reference points use id -1. | +| `TargetValues` | `List` | `[[406, 342, 134], [402, 205, 135], [180, 209, 136]]` | Measured sensor points as [x, y, touchId]. The third entry is an identifier, not depth. | +| `UpperThreshold` | `int` | `1010` | Stored upper threshold for the calibration gradient processor. | +| `LowerThreshold` | `int` | `960` | Stored lower threshold for the calibration gradient processor. | +| `LastUpdated` | `List` | `["2025-04-14T16:17:53.4293293+02:00", "2025-04-14T16:17:58.5862257+02:00", "2025-04-14T16:18:03.7263554+02:00"]` | Update timestamps for calibration points, serialized as ISO date/time strings. | + +__[⬆ back to top](#table-of-contents)__ + +## FrameSize + +Projection/calibration viewport (`FrameSizeDefinition`). + +| Property | Type (.NET) | Example value | Description | +| --- | --- | --- | --- | +| `Width` | `int` | `1920` | Viewport width in pixels. | +| `Height` | `int` | `1080` | Viewport height in pixels. | +| `Left` | `int` | `0` | Horizontal viewport offset in pixels. | +| `Top` | `int` | `0` | Vertical viewport offset in pixels. | + +__[⬆ back to top](#table-of-contents)__ + +## Camera and Startup + +These two properties are at the root of the configuration. + +| Property | Type (.NET) | Example value | Description | +| --- | --- | --- | --- | +| `IsAutoStartEnabled` | `bool` | `true` | Enables automatic startup of tracking. | +| `DefaultCamera` | `string?` | `"Microsoft© Azure Kinect"` | Name of the camera selected for startup. | + +__[⬆ back to top](#table-of-contents)__ + +## CameraConfigurationValues + +Requested camera stream configuration (`CameraConfiguration`). + +| Property | Type (.NET) | Example value | Description | +| --- | --- | --- | --- | +| `Width` | `int` | `640` | Depth stream width in pixels. | +| `Height` | `int` | `576` | Depth stream height in pixels. | +| `Framerate` | `int` | `30` | Frame rate in frames per second. | + +__[⬆ back to top](#table-of-contents)__ + +## NetworkSettingValues + +Transmission of the ReFlex interaction format (`NetworkSettings`). TUIO has separate settings below. + +| Property | Type (.NET) | Example value | Description | +| --- | --- | --- | --- | +| `NetworkInterfaceType` | `NetworkInterface` | `1` | `0`: None; `1`: Websockets; `2`: Tcp. | +| `Interval` | `float` | `30.0` | Broadcast interval in milliseconds. | +| `Address` | `string` | `"127.0.0.1"` | Network address. | +| `Port` | `int` | `40001` | Network port. | +| `Endpoint` | `string` | `"/ReFlex"` | WebSocket endpoint path. | + +__[⬆ back to top](#table-of-contents)__ + +## PointCloudSettingValues + +Point-cloud preview settings. The class defaults match the saved values shown here. + +| Property | Type (.NET) | Example value | Description | +| --- | --- | --- | --- | +| `FullResolution` | `bool` | `false` | Enables the full-resolution point cloud. | +| `UpdateInterval` | `int` | `100` | Point-cloud update interval in milliseconds. | +| `PointCloudSize` | `int` | `40000` | Target point count for the reduced point cloud. | + +__[⬆ back to top](#table-of-contents)__ + +## ProcessingSettingValues + +Interaction processing configuration (`ProcessingSettings`). + +| Property | Type (.NET) | Example value | Description | +| --- | --- | --- | --- | +| `InteractionType` | `ObserverType` | `2` | `0`: None; `1`: SingleTouch; `2`: MultiTouch; `3`: Remote. This selects the observer, not the push/pull interaction type. | +| `IntervalDuration` | `int` | `50` | Processing update interval in milliseconds. | + +__[⬆ back to top](#table-of-contents)__ + +## RemoteProcessingServiceSettingsValues + +Configuration of the remote processing service. Class defaults are `http://localhost:50051/`, `0`, `true`, `0.1`, `1200` and `Default`, respectively. + +| Property | Type (.NET) | Example value | Description | +| --- | --- | --- | --- | +| `Address` | `string` | `"http://localhost:50051"` | Remote processing service URL. | +| `NumSkipValues` | `int` | `0` | Stored sample-skipping parameter; its effect depends on the remote processing implementation. | +| `CompleteDataSet` | `bool` | `false` | Selects transmission of the complete dataset. | +| `CutOff` | `double` | `0.1` | Filter cutoff parameter passed to the remote service. | +| `Factor` | `double` | `1200.0` | Scaling factor passed to the remote service. | +| `Algorithm` | `RemoteProcessingAlgorithm` | `0` | `0`: Default; `1`: FiltFilt. | + +__[⬆ back to top](#table-of-contents)__ + +## TuioSettingValues + +TUIO transmission settings (`TuioConfiguration`). See the [TUIO interface]({{ site.baseurl }}/software/repo/tuio.html) for message formats and coordinate mapping. + +| Property | Type (.NET) | Example value | Description | +| --- | --- | --- | --- | +| `SensorWidth` | `int` | `640` | Sensor width used to compute dimension = SensorWidth × SensorHeight. | +| `SensorHeight` | `int` | `576` | Sensor height used to compute dimension. | +| `SensorDescription` | `string` | `"Microsoft© Azure Kinect"` | Sensor name included in the source description. | +| `SessionId` | `int` | `33` | Session identifier used in point and alive messages. | +| `ServerAddress` | `string` | `"127.0.0.1"` | Address of the TUIO receiver. Class default: 127.0.0.1. | +| `ServerPort` | `int` | `3333` | Receiver port. Class default: 3333. | +| `Protocol` | `ProtocolVersion` | `0` | `0`: TUIO_VERSION_1_1; `1`: TUIO_VERSION_2_0 (class default). | +| `Transport` | `TransportProtocol` | `0` | `0`: Udp (class default); `1`: Tcp; `2`: WebSocket. | +| `Interpretation` | `TuioInterpretation` | `0` | `0`: TouchPoint2DwithPressure (class default); `1`: Point3D. | + +__[⬆ back to top](#table-of-contents)__ + +The [TUIO REST endpoints]({{ site.baseurl }}/software/apps/server/rest-api.html#tuio) provide configuration, start/stop and persistence operations. Broadcasting is toggled with `PUT /api/Tuio/ToggleBroadcast`; the current configuration is available through `GET /api/Tuio/GetTuioConfiguration`. + +The [SignalR interface]({{ site.baseurl }}/software/apps/server/signal-r.html#list-of-signalr-hubs) exposes `/tuiohub`: `startState` subscribes to the broadcast state and `startPackageDetails` subscribes to package details for monitoring. + +__[⬆ back to top](#table-of-contents)__ diff --git a/docs/software/repo/requirements.md b/docs/software/repo/requirements.md index fa94ee94..d96af2b2 100644 --- a/docs/software/repo/requirements.md +++ b/docs/software/repo/requirements.md @@ -1,5 +1,117 @@ --- -title: Requirements Nalaysis +title: Requirements Analysis --- # {{ page.title }} + + +## Table of Contents + +1. [Functional Requirements](#functional-requirements) +2. [Development Tools](#development-tools) +3. [Non-Functional Requirements](#non-functional-requirements) + +The ReFlex framework evolved through the conception, implementation, and demonstration of [Elastic Display case studies]({{ site.baseurl }}/knowledge/case-studies/overview.html). Each application exposed requirements that existing tools did not yet meet. The tools were extended for that scenario, followed by a consolidation step: reusable functionality was integrated into the main framework, more specific mechanisms were generalized, and application-specific components remained outside the core. + +This process distinguishes three kinds of contribution: + +- **Functional requirements** define the capabilities needed to capture, process, configure, and communicate surface deformation. +- **Non-functional requirements** define the expected responsiveness, reliability, architectural flexibility, and sustainability of the software. +- **Development tools** support prototyping, testing, integration, and the creation of new applications. + +The following figure traces these contributions across the case studies and subsequent research projects. It records the origin of requirements and consolidation decisions rather than the complete feature set of each prototype. + +![Requirements derived from the case studies, showing integrated and excluded functionality, non-functional requirements, and developer tools. Blue entries indicate functionality integrated into the framework; grey entries indicate concepts excluded from the core; green entries represent non-functional requirements; orange entries identify development tools.]({{ site.baseurl }}/assets/img/overview/requirement_analysis.png){:.full-width-scheme .transparent-background} + +**[⬆ back to top](#table-of-contents)** + +## Functional Requirements + +The progression began with direct use of surface-wide depth data, added discrete contact detection, and eventually separated tracking from visualization through a distributed architecture. The resulting framework needed to expose both continuous surface representations and individual [interactions]({{ site.baseurl }}/software/repo/interactions.html), with configuration and communication independent of a particular application technology. + +| Case study | Functional contribution | +| ---------- | -------------------------------------- | +| [DepthTouch]({{ site.baseurl }}/knowledge/case-studies/depthtouch.html) | **+** Calculate vector field from surface deformation
**+** Three-point calibration to map sensor coordinates to screen coordinates

**-** physics simulation (application-specific) | +| [FlexiWall]({{ site.baseurl }}/knowledge/case-studies/flexiwall.html) | **+** access to depth image
**+** transform depth image coordinates between sensor and projection spaces
**+** adjust depth values interaction range

**-** Pixel-based layer blending was not included in the framework
**-** Layer data management (application-specific) | +| [DEEP]({{ site.baseurl }}/knowledge/case-studies/deep.html) | **+** Processing of point cloud derived from depth image
**+** Multi-touch detection without persistent Touch IDs
**+** Definition of resting plane and deformation thresholds
**+** Sensor abstraction: Switch between Kinect 1 and 2

**-** Physics simulation (application-specific)
**-** Zoomable UI
**-** Extended calibration (no siginicant improvement, too complicated)| +| [Interactive Animations]({{ site.baseurl }}/knowledge/case-studies/interactive-animations.html) | **+** Single-touch detection as generic functionality
**+** Calibration based on matching projected image
**+** Sensor abstraction: Intel RealSense support
**+** Automatic sensor and application startup

**-** Navigation gestures (application-specific)
**-** Animation control (no general approach) | +| [Zoomable Product Browser]({{ site.baseurl }}/knowledge/case-studies/zoomable-product-browser.html) | **+** Multi-touch with persistent IDs

**-** Gesture recognition (application-specific) | +| [Glyphboard]({{ site.baseurl }}/knowledge/case-studies/glyphboard.html) | **+** Definition of common format for streaming interaction data
**+** Communication via WebSockets

**-** Gesture recognition (application-specific) | +| [DeepZoom]({{ site.baseurl }}/knowledge/case-studies/deep-zoom.html) | **+** Point-cloud configuration (thresholds, distances, limits)
**+** Point-cloud filtering (smoothing, outliers) | +| [DisPlay]({{ site.baseurl }}/knowledge/case-studies/display.html) | **+** Viewport coordinate mapping
**+** user interface for server configuration (point cloud configuration, filtering, interaction processing)
**+** Persisting server settings and parameterizes server on startup by client applications | +| [Construction Progress Visualization]({{ site.baseurl }}/knowledge/case-studies/bim.html) | *No new core functions integrated into framework* | +| [Layers]({{ site.baseurl }}/knowledge/case-studies/layers.html) | **+** Sensor abstraction: Azure Kinect support
**+** Web-based configuration UI
**+** configuration and pipeline control via REST-interface
**+** Interaction streaming via WebSockets
**+** Streaming surface data (point cloud, vector field) via SignalR
**+** Concurrent interaction and depth-image streams

**-** Interactive lenses | +| [Further studies, prototypes, and student projects]({{ site.baseurl }}/knowledge/case-studies/other-prototypes.html#implications-for-framework-development) | **+** Interfaces with other frameworks (TUIO, gRPC)
**+** Interaction logging (e.g. for user studies)
**+** Surface measurements (for reproducction of exact surface streching behaviour )

**-** External actuator control (TactJam) | + +The main architectural consequence was a clear division of responsibilities: the framework captures and processes deformation and exposes the resulting data and configuration interfaces; applications interpret these inputs and derive how they affect content, simulated forces, gestures, and interface state. Persistent identifiers support gestures that require the history of individual contacts. + +**[⬆ back to top](#table-of-contents)** + +## Development Tools + +The tools evolved alongside the input representations and architecture. A synthetic depth image suited FlexiWall's shader-based rendering, but was insufficient for applications expecting extracted contact data. Later tools therefore simulated contacts or a point cloud and supported recording and playback. + +| Case Study | Tool Description | +| ------- | ------------------- | ------- | +| FlexiWall | **Depth-image emulator**: Develop and prepare layered content without requiring the physical installation. | +| DeepZoom | **Touch-point simulator**: Test application behaviour using synthetic interaction points independently of the tracking hardware. | +| DisPlay | **Unreal Engine plugin**: Integrate surface interactions with a game-engine client and support reuse across technologies. | +| Layers | **Point-cloud emulator**: Generate synthetic surface data
Recording/playback: replay captured data for development, testing, and repeatable investigation. | +| Further studies and student projects | **Application templates** for different frontend technologies (HTML/JS, Angular, Vue.js, React, WPF, .NET MAUI, Unity, Unreal Engine): Reduce the effort needed to create clients
**Unity Plugin**: Integrate external technologies
**Developer documentation**: Facilitating Open Source contributions
**Class, type, and component libraries**: Maintainability, ensuring consistent behaviour, look and feel | + +Feedback from these case studies student projects and also established requirements for platform-independent development, a repository structure that supports collaboration, continuous integration with tests, and the preservation of study-specific builds. The [student-project experiences]({{ site.baseurl }}/knowledge/case-studies/other-prototypes.html#implications-for-framework-development) illustrate how missing templates, incomplete documentation, emulator limitations, and integration problems informed subsequent improvements. + +**[⬆ back to top](#table-of-contents)** + +## Non-Functional Requirements + +The non-functional requirements fall into three groups: responsiveness and reliable use; architectural properties; and sustainable collaboration as an open-source research tool. + +| Group | Requirement | Target / measure | +| ----- | ----------- | ---------------- | +| Usability | Performance | Processing in **< 50 ms**, with a **30 ms target**; depth-image streaming with little additional impact on latency. | +| Usability | Availability | Failure-free operation for **at least 4 hours**. | +| Usability | Robustness | Stable memory usage. | +| Architecture | Decoupling UI and tracking | Distributed architecture. | +| Architecture | Configurability | Adaptation through suitable interfaces. | +| Architecture | Extensibility | Modular structure. | +| Architecture | Installability | Installers for tools and applications. | +| Open Source / Open Science | Portability | Use on common desktop operating systems. | +| Open Source / Open Science | Maintainability | Continuous-integration pipeline and tests. | +| Open Source / Open Science | Reusability | Reusable classes and components; application templates. | +| Open Source / Open Science | Suitable repository structure | Monorepository with integrated documentation. | +| Open Source / Open Science | Reproducibility | Specific releases and data repositories. | + +### Responsiveness and Streaming + +The latency budget uses approximately **100 ms for the complete interaction-to-response path** as a design reference for direct manipulation. The source analysis contrasts this with the more permissive reference of approximately one second for navigation where users can see that an operation is being performed. For an Elastic Display's continuous deformation feedback, the stricter reference motivates the framework's processing target. + +Three components contribute to the overall delay: sensor acquisition, framework processing and delivery, and application-side processing and presentation. The sensors considered in the analysis operate at up to **30 Hz**, corresponding to an interval of approximately **33 ms between frames**. The initial budget allocates approximately another 33 ms to the application, leaving roughly 33 ms for framework processing and delivery. From this, the preferred framework target is set to **30 ms**. + +The analysis assumes that transfer from the physically connected sensor adds less than 10 ms and does not budget it separately. This is a simplifying assumption, and the frame interval should not be treated as a measurement of total sensor latency. The **< 50 ms processing limit** is a relaxed bound, motivated by the expectation that client processing is generally less complex than point-cloud analysis. Meeting this bound alone does not guarantee a 100 ms overall response: the sensor, transport, and client delays still need to fit within the remaining budget. Processing and delivery times must therefore be distinguished from the complete interaction-to-response time when validating the system. + +Depth images and point clouds contain much more data than lists of interactions. Their streaming consequently adds a bandwidth requirement as well as a latency requirement. The objective is to keep delivery of surface data as close as possible to the latency of interaction events, including when both streams are active. Layers made this requirement explicit by consuming interaction data and depth images simultaneously. Request-based configuration through the REST API remains separate from continuous data delivery. + +### Availability and Robustness + +Exhibitions require the system to operate predictably for extended periods. The target of at least four hours without failure makes this expectation more concrete than a general demand for stability. Automatic initialization and restoration of configuration support installation setup, while reliable runtime behaviour is required throughout the session. + +Stable memory usage is a prerequisite for such operation. Depth-image and point-cloud processing involve copying sensor values, and parallel pipeline steps can retain several representations at once. Avoiding memory leaks and uncontrolled accumulation of data is therefore important for both sustained performance and failure-free use. A successful short demonstration does not by itself establish that these longer-duration requirements are satisfied. + +### Architecture and Configurability + +The central architectural requirement is the **decoupling of tracking from the user interface**. Applications should be able to use the tools across different scenarios and client technologies without requiring scenario-specific changes to the tracking core. A distributed architecture provides the communication boundary needed for this separation. + +Configurability allows applications and development tools to adapt sensor selection, processing, calibration, viewport mapping, and communication through defined interfaces. This also supports reproducible startup and the transition between scenarios without rebuilding the tracking implementation. + +Extensibility requires a modular structure in which sensors, processing steps, and integration components can be added or exchanged. This follows the open-closed principle: extend functionality through suitable boundaries while limiting changes to established components. Installers complement these architectural measures by making tools and applications easier to deploy. + +### Open Source and Open Science + +Portability targets use of the framework and tools on **Windows, Linux, and macOS desktop systems**. The purpose is to support research and development in different environments. This remains a requirement for the software; compatibility of individual sensors and their SDKs must also be considered when choosing a setup. + +Maintainability and collaborative integration depend on automated builds and tests, supported by the [continuous-integration pipeline]({{ site.baseurl }}/software/repo/ci.html). Reusable libraries, shared components, and application templates reduce duplicated work and help new applications adopt the same interfaces. + +A monorepository with integrated documentation keeps the framework, related tools, and their usage information together. Clear structure and documentation support both reuse and contributions. For Open Science, reproducibility additionally requires identifying and preserving the software build associated with a study, together with the relevant data. Specific releases, archived builds, and data repositories make it possible to distinguish the version used for an experiment from later development. + +**[⬆ back to top](#table-of-contents)** diff --git a/docs/software/repo/tuio.md b/docs/software/repo/tuio.md index e69de29b..561c6389 100644 --- a/docs/software/repo/tuio.md +++ b/docs/software/repo/tuio.md @@ -0,0 +1,165 @@ +--- +title: TUIO Interface +--- + +# {{ page.title }} + + +## Table of Contents + +1. [Introduction](#introduction) +2. [Supported Formats](#supported-formats) +3. [Coordinates and Interaction Mapping](#coordinates-and-interaction-mapping) +4. [TUIO 2.0 Message Specification](#tuio-20-message-specification) +5. [TUIO 1.1 Message Specification](#tuio-11-message-specification) +6. [Message Example (TUIO 2.0)](#message-example-tuio-20) +7. [Configuration and Monitoring](#configuration-and-monitoring) + +## Introduction + +ReFlex supports TUIO as an alternative to the JSON [Interactions Format]({{ site.baseurl }}/software/repo/interactions.html). Detected multi-touch interactions are converted into OSC messages using CoreOSC and sent to a receiving application via UDP, TCP or WebSockets. Both TUIO 1.1 and TUIO 2.0 are supported. + +__[⬆ back to top](#table-of-contents)__ + +## Supported Formats + +Interactions can be encoded as 2D points with an additional pressure component or as 3D points. The interpretation is selected independently of the transport protocol. + +| Interpretation | Configuration value | TUIO 1.1 address | TUIO 2.0 address | Depth representation | +| --- | --- | --- | --- | --- | +| 2D point with pressure | `TouchPoint2DwithPressure` (`0`) | `/tuio/25Dobj` | `/tuio2/ptr` | Signed depth in `[-1, 1]` | +| 3D point | `Point3D` (`1`) | `/tuio/3Dobj` | `/tuio2/p3d` | Depth mapped to `[0, 1]` | + +__[⬆ back to top](#table-of-contents)__ + +## Coordinates and Interaction Mapping + +The TUIO conversion copies `Interaction.Position.X` and `Interaction.Position.Y` without further normalization. Code 7-2 uses calibrated screen coordinates (for example, `292`, `117`); clients must therefore interpret these values in the coordinate system supplied by the processing/calibration pipeline rather than assume that they are always in `[0, 1]`. + +| Interaction property | TUIO representation | Description | +| --- | --- | --- | +| `TouchId` | `c_id` (2.0), class ID `i` (1.1) | Identifies the tracked touch point. | +| `Position.X`, `Position.Y` | `x`, `y` | Lateral position, copied unchanged. | +| `Position.Z` with `TouchPoint2DwithPressure` | Pressure (2.0), `z` (1.1) | Signed normalized depth in `[-1, 1]`. Negative values represent pushing towards the sensor; positive values represent pulling towards the user. Zero is the undeformed surface. | +| `Position.Z` with `Point3D` | `z` | Converted using `(Position.Z + 1) / 2`. The undeformed surface is at `0.5`. | + +`Confidence`, `Type`, `ExtremumDescription`, position validity/filter flags and the interaction's `Time` are not included in these messages. Frame metadata is generated separately. + +ReFlex uses the configured `SessionId` for all points in a frame and for the alive message. Individual touches are distinguished by their touch/class IDs; the alive message does not enumerate individual touch IDs. + +__[⬆ back to top](#table-of-contents)__ + +## TUIO 2.0 Message Specification + +Each frame starts with `/tuio2/frm`, followed by one `/tuio2/ptr` or `/tuio2/p3d` message per interaction, and ends with `/tuio2/alv`. + +### Frame (`/tuio2/frm`) + +| Argument | Type (.NET / CoreOSC) | Description | +| --- | --- | --- | +| `frame_id` | `Int32` | Frame identifier for ordering updates. | +| `time` | `Timetag` | Constructed by ReFlex from `DateTime.Now.Ticks`: the number of 100-nanosecond intervals since January 1, 0001. This is an implementation-specific .NET tick value wrapped in an OSC timetag; it must not be interpreted as a conventional OSC/NTP timestamp. | +| `dimension` | `Int32` | Computed as `SensorWidth * SensorHeight`, e.g. `640 * 480 = 307200`. | +| `source` | `String` | Source description. The current implementation produces `ReFlex@{clientAddress} [{SensorDescription}]`. | + +### 2D Point with Pressure (`/tuio2/ptr`) + +Arguments are transmitted in the following order: + +| Argument | Type (.NET) | Description | +| --- | --- | --- | +| `session_id` | `Int32` | Configured session ID. | +| `tu_id` | `Int32` | Constant `1`, used by ReFlex for the finger type. | +| `c_id` | `Int32` | Interaction `TouchId`. | +| `x` | `Single` | Horizontal position. | +| `y` | `Single` | Vertical position. | +| `angle` | `Single` | Constant `0`. | +| `shear` | `Single` | Constant `0`. | +| `radius` | `Single` | Constant `1`. | +| `pressure` | `Single` | Signed normalized depth in `[-1, 1]`. | + +### 3D Point (`/tuio2/p3d`) + +| Argument | Type (.NET) | Description | +| --- | --- | --- | +| `session_id` | `Int32` | Configured session ID. | +| `tu_id` | `Int32` | Constant `1`. | +| `c_id` | `Int32` | Interaction `TouchId`. | +| `x`, `y`, `z` | `Single` each | Position; `z` is mapped to `[0, 1]`. | +| `x_axis`, `y_axis`, `z_axis` | `Single` each | Constant direction `(0, 0, 1)`. | +| `radius` | `Single` | Constant `1`. | + +### Alive (`/tuio2/alv`) + +The final message contains one `Int32` argument: the configured `session_id`. + +__[⬆ back to top](#table-of-contents)__ + +## TUIO 1.1 Message Specification + +All messages in a frame use the selected profile address (`/tuio/25Dobj` or `/tuio/3Dobj`). The first argument is a string command: + +| Order | Command | Remaining arguments | Description | +| --- | --- | --- | --- | +| 1 | `source` | `source` (`String`) | Source description, as in TUIO 2.0. | +| 2 | `alive` | `session_id` (`Int32`) | Configured session ID. | +| 3…n | `set` | See below | One message per interaction. | +| Last | `fseq` | `frame_id` (`Int32`) | Frame identifier. | + +The `set` arguments are ordered as follows. `s` is the session ID and `i` is the touch ID (both `Int32`); all subsequent arguments are `Single` values. + +{% highlight text linenos %} +/tuio/25Dobj set s i x y z a X Y Z A m r +/tuio/3Dobj set s i x y z a b c X Y Z A B C m r +{% endhighlight %} + +In the 2.5D profile, `z` carries signed depth; in the 3D profile, it carries remapped depth. ReFlex sets all angle (`a`, `b`, `c`), velocity (`X`, `Y`, `Z`), rotational velocity (`A`, `B`, `C`) and acceleration (`m`, `r`) values to `0`. + +__[⬆ back to top](#table-of-contents)__ + +## Message Example (TUIO 2.0) + +The following code example contains two points encoded as 2D points with pressure. The following is a readable representation of the OSC messages, with decimal points used for numeric values. The source string is preserved from the listing; the current implementation adds brackets around the sensor description. + +{% highlight text linenos %} +[1] /tuio2/frm + 59671 : Int32 (frame_id) + 638875034280336600 : Timetag (time) + 307200 : Int32 (dimension) + ReFlex@::1 Azure Kinect : String (source) + +[2] /tuio2/ptr + 33 : Int32 (session_id) + 1 : Int32 (tu_id) + 2 : Int32 (c_id / TouchId) + 292 : Single (x) + 117 : Single (y) + 0 : Single (angle) + 0 : Single (shear) + 1 : Single (radius) + -0.13418955 : Single (pressure) + +[3] /tuio2/ptr + 33 : Int32 (session_id) + 1 : Int32 (tu_id) + 3 : Int32 (c_id / TouchId) + 278 : Single (x) + 227 : Single (y) + 0 : Single (angle) + 0 : Single (shear) + 1 : Single (radius) + -0.106472194 : Single (pressure) + +[4] /tuio2/alv + 33 : Int32 (session_id) +{% endhighlight %} + +The two touches have IDs `2` and `3`, share session `33`, and both represent pushes into the surface. Their positions are calibrated screen coordinates; the pressure values represent normalized deformation depth. + +__[⬆ back to top](#table-of-contents)__ + +## Configuration and Monitoring + +Configuration fields, saved example values, defaults and monitoring endpoints are documented under [TuioSettingValues in the configuration file]({{ site.baseurl }}/software/repo/config-file.html#tuiosettingvalues). + +__[⬆ back to top](#table-of-contents)__