diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 21ecd0eab3..a3d57c109d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -83,6 +83,8 @@ jobs: timeout-minutes: 30 outputs: version: ${{ steps.version.outputs.version }} + elasticsearch_image_tag: ${{ steps.elasticsearch_image.outputs.tag }} + build_elasticsearch_image: ${{ steps.elasticsearch_image.outputs.build_locally }} should_publish: ${{ (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'dev-preview')) && secrets.DOCKER_USERNAME != '' && secrets.DOCKER_PASSWORD != '' }} is_prod_deploy: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name != 'pull_request' }} is_dev_deploy: ${{ (github.event_name == 'repository_dispatch' && github.event.action == 'preview') || (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'dev-preview')) }} @@ -129,9 +131,65 @@ jobs: echo "version=$version" >> $GITHUB_OUTPUT echo "### $version" >> $GITHUB_STEP_SUMMARY + - name: Resolve Elasticsearch image + id: elasticsearch_image + env: + PR_HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name }} + PREVIEW_BASE_SHA: ${{ github.event.client_payload.base_sha }} + PUSH_BEFORE_SHA: ${{ github.event.before }} + run: | + version=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' build/docker/elasticsearch/9.x/Dockerfile) + tag=$version + image_changed=false + build_locally=false + + if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]] && + ! git diff --quiet origin/main...HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then + image_changed=true + elif [[ "$GITHUB_EVENT_NAME" == "push" && "$GITHUB_REF" == "refs/heads/main" ]] && + ! git diff --quiet "$PUSH_BEFORE_SHA"..HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then + image_changed=true + elif [[ "$GITHUB_EVENT_NAME" == "repository_dispatch" ]] && + ! git diff --quiet "${PREVIEW_BASE_SHA:-origin/main}"...HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then + image_changed=true + elif [[ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]] && + ! git diff --quiet origin/main...HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then + image_changed=true + fi + + if [[ "$image_changed" == "true" ]]; then + image_sha=$(git ls-files -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml | sort | xargs sha256sum | sha256sum | cut -d " " -f 1) + tag="$version-sha256-$image_sha" + image="exceptionless/elasticsearch:$tag" + + if [[ "$GITHUB_EVENT_NAME" == "pull_request" && "$PR_HEAD_REPOSITORY" != "$GITHUB_REPOSITORY" ]]; then + build_locally=true + else + for attempt in {1..150}; do + if docker manifest inspect "$image" > /dev/null 2>&1; then + break + fi + + if [[ "$attempt" -eq 150 ]]; then + echo "::error::Timed out waiting for $image to be published." + exit 1 + fi + + sleep 10 + done + fi + fi + + echo "tag=$tag" >> "$GITHUB_OUTPUT" + echo "build_locally=$build_locally" >> "$GITHUB_OUTPUT" + echo "### Elasticsearch image: exceptionless/elasticsearch:$tag" >> "$GITHUB_STEP_SUMMARY" + test-api: + needs: version runs-on: ubuntu-latest timeout-minutes: 30 + env: + Elasticsearch__ImageTag: ${{ needs.version.outputs.elasticsearch_image_tag }} steps: - name: Checkout @@ -139,6 +197,10 @@ jobs: with: ref: ${{ (github.event_name == 'repository_dispatch' && github.event.client_payload.head_sha) || github.event.pull_request.head.sha || github.sha }} + - name: Build fork Elasticsearch candidate locally + if: ${{ needs.version.outputs.build_elasticsearch_image == 'true' }} + run: docker build --tag "exceptionless/elasticsearch:${{ needs.version.outputs.elasticsearch_image_tag }}" --file build/docker/elasticsearch/9.x/Dockerfile build/docker/elasticsearch/9.x + - name: Setup .NET Core uses: actions/setup-dotnet@v6 with: @@ -223,12 +285,21 @@ jobs: run: echo "npm run test:integration" test-e2e: + needs: version runs-on: ubuntu-latest timeout-minutes: 45 + env: + Elasticsearch__ImageTag: ${{ needs.version.outputs.elasticsearch_image_tag }} steps: - name: Checkout uses: actions/checkout@v6 + with: + ref: ${{ (github.event_name == 'repository_dispatch' && github.event.client_payload.head_sha) || github.event.pull_request.head.sha || github.sha }} + + - name: Build fork Elasticsearch candidate locally + if: ${{ needs.version.outputs.build_elasticsearch_image == 'true' }} + run: docker build --tag "exceptionless/elasticsearch:${{ needs.version.outputs.elasticsearch_image_tag }}" --file build/docker/elasticsearch/9.x/Dockerfile build/docker/elasticsearch/9.x - name: Setup .NET Core uses: actions/setup-dotnet@v6 @@ -290,8 +361,8 @@ jobs: - name: Wait for Aspire Resources run: | for attempt in {1..60}; do - if curl -fksS https://web-ex.dev.localhost:7131/api/v2/about > /dev/null && - curl -fksS https://web-ex.dev.localhost:7131/next/login > /dev/null; then + if curl -fksS --connect-timeout 5 --max-time 10 https://web-ex.dev.localhost:7131/api/v2/about > /dev/null && + curl -fksS --connect-timeout 5 --max-time 10 https://web-ex.dev.localhost:7131/next/login > /dev/null; then break fi @@ -310,8 +381,8 @@ jobs: - name: Verify E2E Endpoints run: | - curl -fksS https://web-ex.dev.localhost:7131/api/v2/about > /dev/null - curl -fksS https://web-ex.dev.localhost:7131/next/login > /dev/null + curl -fksS --connect-timeout 5 --max-time 10 https://web-ex.dev.localhost:7131/api/v2/about > /dev/null + curl -fksS --connect-timeout 5 --max-time 10 https://web-ex.dev.localhost:7131/next/login > /dev/null - name: Run Playwright E2E Tests working-directory: src/Exceptionless.Web/ClientApp diff --git a/.github/workflows/elasticsearch-docker-8.yml b/.github/workflows/elasticsearch-docker-8.yml index 6f65a4ccd6..f8577d97c8 100644 --- a/.github/workflows/elasticsearch-docker-8.yml +++ b/.github/workflows/elasticsearch-docker-8.yml @@ -43,4 +43,4 @@ jobs: working-directory: build/docker/elasticsearch/8.x run: | VERSION=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' Dockerfile) - docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file ./Dockerfile . --tag exceptionless/elasticsearch:$VERSION --tag exceptionless/elasticsearch:latest + docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file ./Dockerfile . --tag exceptionless/elasticsearch:$VERSION diff --git a/.github/workflows/elasticsearch-docker-9.yml b/.github/workflows/elasticsearch-docker-9.yml new file mode 100644 index 0000000000..a911616691 --- /dev/null +++ b/.github/workflows/elasticsearch-docker-9.yml @@ -0,0 +1,52 @@ +name: Elasticsearch 9.x Docker Image CI + +on: + push: + paths: + - "build/docker/elasticsearch/9.x/**" + - ".github/workflows/elasticsearch-docker-9.yml" + +jobs: + build: + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/v') != true + + steps: + - uses: actions/checkout@v7 + - name: Setup .NET Core + uses: actions/setup-dotnet@v5 + with: + dotnet-version: 10.0.301 + - name: Build Reason + env: + GITHUB_EVENT: ${{ toJson(github) }} + run: "echo ref: ${{github.ref}} event: ${{github.event_name}}" + - name: Build Version + run: | + dotnet tool install --global minver-cli --version 7.0.0 + version=$(minver --tag-prefix v) + echo "MINVERVERSIONOVERRIDE=$version" >> $GITHUB_ENV + echo "VERSION=$version" >> $GITHUB_ENV + echo "### Version: $version" >> $GITHUB_STEP_SUMMARY + - name: Set up QEMU + uses: docker/setup-qemu-action@v4 + - name: Login to DockerHub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + with: + platforms: linux/amd64,linux/arm64 + - name: Build custom Elasticsearch 9.x docker image + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + VERSION=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' build/docker/elasticsearch/9.x/Dockerfile) + IMAGE_SHA=$(git ls-files -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml | sort | xargs sha256sum | sha256sum | cut -d " " -f 1) + TAGS=(--tag "exceptionless/elasticsearch:$VERSION-sha256-$IMAGE_SHA") + if [[ "$GITHUB_REF" == "refs/heads/$DEFAULT_BRANCH" ]]; then + TAGS+=(--tag "exceptionless/elasticsearch:$VERSION" --tag "exceptionless/elasticsearch:latest") + fi + docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file build/docker/elasticsearch/9.x/Dockerfile build/docker/elasticsearch/9.x "${TAGS[@]}" diff --git a/.github/workflows/preview-command.yml b/.github/workflows/preview-command.yml index fe94875393..e4dd285c7f 100644 --- a/.github/workflows/preview-command.yml +++ b/.github/workflows/preview-command.yml @@ -77,6 +77,7 @@ jobs: const headRepository = pullRequest.head.repo.full_name; const headRef = pullRequest.head.ref; const headSha = pullRequest.head.sha; + const baseSha = pullRequest.base.sha; const headLabel = pullRequest.head.label; const headShortSha = headSha.slice(0, 12); @@ -95,6 +96,7 @@ jobs: core.setOutput("head-ref", headRef); core.setOutput("head-label", headLabel); core.setOutput("head-sha", headSha); + core.setOutput("base-sha", baseSha); core.setOutput("head-short-sha", headShortSha); const previewLabel = "dev-preview"; @@ -142,6 +144,7 @@ jobs: HEAD_REF: ${{ steps.preview.outputs.head-ref }} HEAD_LABEL: ${{ steps.preview.outputs.head-label }} HEAD_SHA: ${{ steps.preview.outputs.head-sha }} + BASE_SHA: ${{ steps.preview.outputs.base-sha }} with: script: | await github.rest.repos.createDispatchEvent({ @@ -152,7 +155,8 @@ jobs: pr_number: Number(process.env.PR_NUMBER), head_ref: process.env.HEAD_REF, head_label: process.env.HEAD_LABEL, - head_sha: process.env.HEAD_SHA + head_sha: process.env.HEAD_SHA, + base_sha: process.env.BASE_SHA } }); diff --git a/Dockerfile b/Dockerfile index 80cc955548..1d1bb7bfa9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -100,7 +100,7 @@ ENTRYPOINT ["/app/app-docker-entrypoint.sh"] # completely self-contained -FROM exceptionless/elasticsearch:8.19.15 AS exceptionless +FROM exceptionless/elasticsearch:9.5.0 AS exceptionless WORKDIR /app COPY --from=job-publish /app/src/Exceptionless.Job/out ./ @@ -113,21 +113,23 @@ COPY ./build/supervisord.conf /etc/ USER root # install dotnet and supervisor -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - supervisor \ +RUN microdnf install -y \ wget \ dos2unix \ ca-certificates \ + python3-pip \ \ # .NET dependencies - libc6 \ - libgcc-s1 \ - libicu74 \ - libssl3 \ - libstdc++6 \ + glibc \ + gzip \ + libgcc \ + libicu \ + openssl-libs \ + libstdc++ \ + tar \ tzdata \ - && rm -rf /var/lib/apt/lists/* \ + && pip3 install --no-cache-dir supervisor==4.3.0 \ + && microdnf clean all \ && dos2unix /app/docker-entrypoint.sh ENV discovery.type=single-node \ diff --git a/build/docker/elasticsearch/9.x/Dockerfile b/build/docker/elasticsearch/9.x/Dockerfile new file mode 100644 index 0000000000..cc03ebce5a --- /dev/null +++ b/build/docker/elasticsearch/9.x/Dockerfile @@ -0,0 +1,4 @@ +# https://www.docker.elastic.co/ +FROM docker.elastic.co/elasticsearch/elasticsearch:9.5.0 + +RUN elasticsearch-plugin install -b mapper-size diff --git a/docker/docker-compose.apm.yml b/docker/docker-compose.apm.yml index bd55972eb9..220098afad 100644 --- a/docker/docker-compose.apm.yml +++ b/docker/docker-compose.apm.yml @@ -2,7 +2,7 @@ version: "2.2" services: setup: - image: docker.elastic.co/elasticsearch/elasticsearch:8.19.15 + image: docker.elastic.co/elasticsearch/elasticsearch:9.5.0 volumes: - certs:/usr/share/elasticsearch/config/certs user: "0" @@ -53,7 +53,7 @@ services: depends_on: setup: condition: service_healthy - image: docker.elastic.co/elasticsearch/elasticsearch:8.19.15 + image: docker.elastic.co/elasticsearch/elasticsearch:9.5.0 volumes: - certs:/usr/share/elasticsearch/config/certs - esdata:/usr/share/elasticsearch/data @@ -98,7 +98,7 @@ services: depends_on: elasticsearch: condition: service_healthy - image: docker.elastic.co/kibana/kibana:8.19.15 + image: docker.elastic.co/kibana/kibana:9.5.0 volumes: - certs:/usr/share/kibana/config/certs ports: @@ -124,7 +124,7 @@ services: depends_on: elasticsearch: condition: service_healthy - image: docker.elastic.co/apm/apm-server:8.19.15 + image: docker.elastic.co/apm/apm-server:9.5.0 volumes: - certs:/usr/share/apm-server/certs ports: diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index a18d81b4cf..2b8518034b 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -50,7 +50,7 @@ services: - appdata:/app/storage elasticsearch: - image: exceptionless/elasticsearch:8.19.15 + image: exceptionless/elasticsearch:9.5.0 environment: discovery.type: single-node xpack.security.enabled: "false" @@ -59,6 +59,8 @@ services: - 9200:9200 - 9300:9300 volumes: + # Complete the Elasticsearch 8.19 upgrade preflight before reusing this volume with Elasticsearch 9. + # See https://exceptionless.com/docs/self-hosting/upgrading-self-hosted-instance - esdata7:/usr/share/elasticsearch/data healthcheck: test: @@ -74,7 +76,7 @@ services: kibana: depends_on: - elasticsearch - image: docker.elastic.co/kibana/kibana:8.19.15 + image: docker.elastic.co/kibana/kibana:9.5.0 ports: - 5601:5601 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index c81594d41f..b1706174b1 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,6 +1,6 @@ services: elasticsearch: - image: exceptionless/elasticsearch:8.19.15 + image: exceptionless/elasticsearch:9.5.0 environment: node.name: elasticsearch cluster.name: exceptionless @@ -11,6 +11,8 @@ services: ports: - 9200:9200 volumes: + # Complete the Elasticsearch 8.19 upgrade preflight before reusing this volume with Elasticsearch 9. + # See https://exceptionless.com/docs/self-hosting/upgrading-self-hosted-instance - esdata:/usr/share/elasticsearch/data healthcheck: test: @@ -26,7 +28,7 @@ services: kibana: depends_on: - elasticsearch - image: docker.elastic.co/kibana/kibana:8.19.15 + image: docker.elastic.co/kibana/kibana:9.5.0 environment: xpack.security.enabled: "false" ports: diff --git a/docs/docs/self-hosting/upgrading-self-hosted-instance.md b/docs/docs/self-hosting/upgrading-self-hosted-instance.md index dd0974989f..2335c9e421 100644 --- a/docs/docs/self-hosting/upgrading-self-hosted-instance.md +++ b/docs/docs/self-hosting/upgrading-self-hosted-instance.md @@ -8,6 +8,41 @@ title: "Upgrading" **If you are upgrading from v1 or [v2](https://github.com/exceptionless/Exceptionless/releases/tag/v2.0.0) you will need to upgrade to [v3.0](https://github.com/exceptionless/Exceptionless/releases/tag/v3.0.0) before upgrading to the latest release.** +## Upgrading from v8 to v9 + +Exceptionless v9 uses Elasticsearch 9. Do not point an Elasticsearch 9 node at an existing data volume until the cluster has been prepared with Elasticsearch 8.19. Elasticsearch 9 can fail to start when incompatible indices created before Elasticsearch 8 remain. + +Use this upgrade path for an existing self-hosted installation: + +1. Take a current Elasticsearch snapshot or other verified backup and test that it can be restored. Elasticsearch does not support downgrading a data directory after it has been upgraded. +2. Upgrade Elasticsearch and Kibana to the latest 8.19.x patch release first, using the existing data volume. Do not start Elasticsearch 9 yet. +3. Stop the Exceptionless app and job services, but leave Elasticsearch and Kibana 8.19 running. This prevents writes while legacy indices are reindexed. +4. Open Kibana's **Upgrade Assistant** and resolve every critical issue. Reindex every active Exceptionless index created before Elasticsearch 8. Delete only indices you have confirmed are no longer needed; do not mark active Exceptionless indices as read-only. +5. If this data volume previously ran Elasticsearch 7, temporarily disable the GeoIP downloader while still on Elasticsearch 8.19. Elasticsearch deletes its downloaded `.geoip_databases` system index when this setting is disabled; Exceptionless data is not affected. + + ```bash + curl -fsS -X PUT "http://localhost:9200/_cluster/settings" \ + -H "Content-Type: application/json" \ + -d '{"persistent":{"ingest.geoip.downloader.enabled":false}}' + ``` + +6. Confirm that the deprecation API reports no critical issues: + + ```bash + curl -fsS "http://localhost:9200/_migration/deprecations?pretty" + ``` + +7. Stop Elasticsearch and Kibana 8.19 without deleting their data volume. Update the Elasticsearch and Kibana images to the v9 versions, start them, and verify cluster health before restarting the Exceptionless app and jobs. In Docker Compose, do not run `docker compose down -v` because `-v` deletes the data volume. +8. After Elasticsearch 9 is healthy, restore the default GeoIP downloader behavior: + + ```bash + curl -fsS -X PUT "http://localhost:9200/_cluster/settings" \ + -H "Content-Type: application/json" \ + -d '{"persistent":{"ingest.geoip.downloader.enabled":null}}' + ``` + +See Elastic's [prepare-to-upgrade guide](https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade) for the supported 8.x to 9.x upgrade requirements and Upgrade Assistant details. + ## Upgrading from v7.1 to v8 We simplified the self hosting process by integrating the UI into the existing app images. As such `exceptionless/ui` docker images are deprecated and we recommend using `exceptionless/app`. diff --git a/k8s/elastic-monitor.yaml b/k8s/elastic-monitor.yaml index 943cde867d..f1c47ad253 100644 --- a/k8s/elastic-monitor.yaml +++ b/k8s/elastic-monitor.yaml @@ -4,7 +4,7 @@ metadata: name: elastic-monitor namespace: elastic-system spec: - version: 8.19.15 + version: 9.5.0 podDisruptionBudget: {} nodeSets: - name: main @@ -228,7 +228,7 @@ metadata: name: kibana-monitor namespace: elastic-system spec: - version: 8.19.15 + version: 9.5.0 count: 1 http: tls: @@ -364,7 +364,7 @@ metadata: name: fleet-server namespace: elastic-system spec: - version: 8.19.15 + version: 9.5.0 kibanaRef: name: kibana-monitor elasticsearchRefs: @@ -388,7 +388,7 @@ metadata: name: elastic-agent namespace: elastic-system spec: - version: 8.19.15 + version: 9.5.0 kibanaRef: name: kibana-monitor fleetServerRef: diff --git a/k8s/ex-dev-elasticsearch.yaml b/k8s/ex-dev-elasticsearch.yaml index 46f14cc5eb..6a8b1fd851 100644 --- a/k8s/ex-dev-elasticsearch.yaml +++ b/k8s/ex-dev-elasticsearch.yaml @@ -14,8 +14,8 @@ metadata: name: ex-dev namespace: ex-dev spec: - version: 8.19.15 - image: exceptionless/elasticsearch:8.19.15 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch + version: 9.5.0 + image: exceptionless/elasticsearch:9.5.0 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch secureSettings: - secretName: ex-dev-snapshots http: @@ -68,7 +68,7 @@ metadata: name: ex-dev namespace: ex-dev spec: - version: 8.19.15 + version: 9.5.0 count: 1 elasticsearchRef: name: ex-dev diff --git a/k8s/ex-prod-elasticsearch.yaml b/k8s/ex-prod-elasticsearch.yaml index f159ae3046..de7848bce1 100644 --- a/k8s/ex-prod-elasticsearch.yaml +++ b/k8s/ex-prod-elasticsearch.yaml @@ -14,8 +14,8 @@ metadata: name: ex-prod namespace: ex-prod spec: - version: 8.19.15 - image: exceptionless/elasticsearch:8.19.15 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch + version: 9.5.0 + image: exceptionless/elasticsearch:9.5.0 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch monitoring: metrics: elasticsearchRefs: @@ -79,7 +79,7 @@ metadata: name: ex-prod namespace: ex-prod spec: - version: 8.19.15 + version: 9.5.0 count: 1 elasticsearchRef: name: ex-prod diff --git a/k8s/exceptionless/values.yaml b/k8s/exceptionless/values.yaml index 9b9febe39b..8018140e5f 100644 --- a/k8s/exceptionless/values.yaml +++ b/k8s/exceptionless/values.yaml @@ -35,7 +35,7 @@ elasticsearch: connectionString: image: repository: exceptionless/elasticsearch - tag: 8.19.15 + tag: 9.5.0 pullPolicy: IfNotPresent redis: diff --git a/samples/docker-compose.all-in-one.yml b/samples/docker-compose.all-in-one.yml index 5b1caf2d42..622e74830e 100644 --- a/samples/docker-compose.all-in-one.yml +++ b/samples/docker-compose.all-in-one.yml @@ -20,7 +20,7 @@ services: kibana: depends_on: - elasticsearch - image: docker.elastic.co/kibana/kibana:8.19.15 + image: docker.elastic.co/kibana/kibana:9.5.0 ports: - 5601:5601 diff --git a/samples/docker-compose.yml b/samples/docker-compose.yml index d73e0518c9..7fcb4cc0de 100644 --- a/samples/docker-compose.yml +++ b/samples/docker-compose.yml @@ -44,7 +44,7 @@ services: - ex_appdata:/app/storage elasticsearch: - image: exceptionless/elasticsearch:8.19.15 + image: exceptionless/elasticsearch:9.5.0 environment: discovery.type: single-node xpack.security.enabled: "false" @@ -58,7 +58,7 @@ services: kibana: depends_on: - elasticsearch - image: docker.elastic.co/kibana/kibana:8.19.15 + image: docker.elastic.co/kibana/kibana:9.5.0 ports: - 5601:5601 diff --git a/src/Exceptionless.AppHost/Exceptionless.AppHost.csproj b/src/Exceptionless.AppHost/Exceptionless.AppHost.csproj index 16cc44ac96..d2a4d8d9ff 100644 --- a/src/Exceptionless.AppHost/Exceptionless.AppHost.csproj +++ b/src/Exceptionless.AppHost/Exceptionless.AppHost.csproj @@ -12,7 +12,6 @@ - diff --git a/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs b/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs index ec5c5e4031..9a7762d4dc 100644 --- a/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs +++ b/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs @@ -1,4 +1,4 @@ -using Elastic.Clients.Elasticsearch; +using System.Text.Json; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; @@ -13,7 +13,7 @@ public static class ElasticsearchBuilderExtensions private const int KibanaPort = 5601; /// - /// Adds a Elasticsearch container to the application model. The default image is "docker.elastic.co/elasticsearch/elasticsearch". This version the package defaults to the 8.19.15 tag of the Elasticsearch container image + /// Adds a Elasticsearch container to the application model. The default image is "docker.elastic.co/elasticsearch/elasticsearch". This version the package defaults to the 9.5.0 tag of the Elasticsearch container image /// /// The . /// The name of the resource. This name will be used as the connection string name when referenced in a dependency. @@ -60,7 +60,11 @@ public static IResourceBuilder AddElasticsearch(this IDis .PublishAsConnectionString(); } - public static IResourceBuilder WithKibana(this IResourceBuilder builder, Action>? configureContainer = null, string? containerName = null) + public static IResourceBuilder WithKibana( + this IResourceBuilder builder, + Action>? configureContainer = null, + string? containerName = null, + int? port = null) { ArgumentNullException.ThrowIfNull(builder); @@ -79,7 +83,7 @@ public static IResourceBuilder WithKibana(this IResourceB var resourceBuilder = builder.ApplicationBuilder.AddResource(resource) .WithImage(ElasticsearchContainerImageTags.KibanaImage, ElasticsearchContainerImageTags.Tag) .WithImageRegistry(ElasticsearchContainerImageTags.KibanaRegistry) - .WithHttpEndpoint(targetPort: KibanaPort, name: containerName) + .WithHttpEndpoint(targetPort: KibanaPort, port: port, name: containerName) .WithUrlForEndpoint(containerName, u => u.DisplayText = "Kibana") .WithEnvironment("xpack.security.enabled", "false") .WithEnvironment(ctx => @@ -121,7 +125,7 @@ internal static class ElasticsearchContainerImageTags public const string Image = "exceptionless/elasticsearch"; public const string KibanaRegistry = "docker.elastic.co"; public const string KibanaImage = "kibana/kibana"; - public const string Tag = "8.19.15"; + public const string Tag = "9.5.0"; } internal sealed class ElasticsearchConnectionHealthCheck(Func connectionStringFactory) : IHealthCheck @@ -132,11 +136,25 @@ public async Task CheckHealthAsync(HealthCheckContext context if (string.IsNullOrEmpty(connectionString)) return new HealthCheckResult(context.Registration.FailureStatus, "Connection string not available."); - using var settings = new ElasticsearchClientSettings(new Uri(connectionString)); - var client = new ElasticsearchClient(settings); - var response = await client.PingAsync(cancellationToken); - return response.IsValidResponse - ? HealthCheckResult.Healthy() - : new HealthCheckResult(context.Registration.FailureStatus, $"Elasticsearch ping failed: {response.DebugInformation}"); + try + { + using var client = new HttpClient { BaseAddress = new Uri(connectionString), Timeout = TimeSpan.FromSeconds(10) }; + using var response = await client.GetAsync("_cluster/health?wait_for_status=yellow&timeout=5s", cancellationToken); + if (!response.IsSuccessStatusCode) + return new HealthCheckResult(context.Registration.FailureStatus, $"Elasticsearch cluster health returned HTTP {(int)response.StatusCode}."); + + await using var responseStream = await response.Content.ReadAsStreamAsync(cancellationToken); + using var document = await JsonDocument.ParseAsync(responseStream, cancellationToken: cancellationToken); + bool timedOut = document.RootElement.TryGetProperty("timed_out", out var timedOutElement) && timedOutElement.GetBoolean(); + string? status = document.RootElement.TryGetProperty("status", out var statusElement) ? statusElement.GetString() : null; + if (!timedOut && status is "yellow" or "green") + return HealthCheckResult.Healthy(); + + return new HealthCheckResult(context.Registration.FailureStatus, $"Elasticsearch cluster health check failed. Timed out: {timedOut}; status: {status ?? "unknown"}."); + } + catch (Exception ex) when (ex is HttpRequestException or JsonException or TaskCanceledException) + { + return new HealthCheckResult(context.Registration.FailureStatus, "Elasticsearch cluster health check request failed.", ex); + } } } diff --git a/src/Exceptionless.AppHost/Program.cs b/src/Exceptionless.AppHost/Program.cs index b12b48aefe..ef33ee8333 100644 --- a/src/Exceptionless.AppHost/Program.cs +++ b/src/Exceptionless.AppHost/Program.cs @@ -13,6 +13,13 @@ bool servicesOnly = HasArgument("--services-only"); bool ciE2E = HasArgument("--ci-e2e"); bool includeDevTools = !ciE2E; +int elasticsearchPort = GetPort("Elasticsearch:Port", 9200); +string elasticsearchImageTag = builder.Configuration["Elasticsearch:ImageTag"] ?? ElasticsearchContainerImageTags.Tag; +string kibanaImageTag = builder.Configuration["Elasticsearch:KibanaImageTag"] ?? ElasticsearchContainerImageTags.Tag; +string elasticsearchContainerName = builder.Configuration["Elasticsearch:ContainerName"] ?? "Exceptionless-Elasticsearch"; +string elasticsearchDataVolume = builder.Configuration["Elasticsearch:DataVolume"] ?? "exceptionless.data.v1"; +string kibanaContainerName = builder.Configuration["Elasticsearch:KibanaContainerName"] ?? "Exceptionless-Kibana"; +int kibanaPort = GetPort("Elasticsearch:KibanaPort", 5601); int oldAppHttpPort = worktreePorts?.OldAppHttp ?? 7120; int oldAppPort = worktreePorts?.OldAppHttps ?? 7121; int oldAppLiveReloadPort = worktreePorts?.OldAppLiveReload ?? 35729; @@ -24,8 +31,9 @@ string exceptionlessServerUrl = worktreePorts?.ApiHttpsUrl ?? $"https://api-ex.dev.localhost:{DefaultApiHttpsPort}"; const string SharedEmailConnectionString = "smtp://localhost:1026"; -var elastic = builder.AddElasticsearch("Elasticsearch", port: 9200) - .WithDataVolume("exceptionless.data.v1") +var elastic = builder.AddElasticsearch("Elasticsearch", port: elasticsearchPort) + .WithImageTag(elasticsearchImageTag) + .WithDataVolume(elasticsearchDataVolume) .WithEndpointProxySupport(false); var storage = builder.AddAzureStorage("Storage") @@ -70,15 +78,17 @@ var ownedElastic = elastic; elastic = ownedElastic .WithLifetime(ContainerLifetime.Persistent) - .WithContainerName("Exceptionless-Elasticsearch"); + .WithContainerName(elasticsearchContainerName); if (!servicesOnly && includeDevTools) { elastic = elastic.WithKibana(b => b + .WithImageTag(kibanaImageTag) .WithLifetime(ContainerLifetime.Persistent) .WithEndpointProxySupport(false) - .WithContainerName("Exceptionless-Kibana") - .WithParentRelationship(ownedElastic)); + .WithContainerName(kibanaContainerName) + .WithParentRelationship(ownedElastic), + port: kibanaPort); } var ownedCache = cache; @@ -242,3 +252,15 @@ await builder.Build().RunAsync(); bool HasArgument(string name) => args.Any(arg => StringComparer.OrdinalIgnoreCase.Equals(arg, name) || StringComparer.OrdinalIgnoreCase.Equals(arg, name.TrimStart('-'))); + +int GetPort(string key, int defaultValue) +{ + string? value = builder.Configuration[key]; + if (String.IsNullOrWhiteSpace(value)) + return defaultValue; + + if (!Int32.TryParse(value, out int port) || port is < 1 or > 65535) + throw new InvalidOperationException($"Configuration value '{key}' must be a valid TCP port."); + + return port; +} diff --git a/src/Exceptionless.AppHost/appsettings.Development.json b/src/Exceptionless.AppHost/appsettings.Development.json index 0c208ae918..832d220e44 100644 --- a/src/Exceptionless.AppHost/appsettings.Development.json +++ b/src/Exceptionless.AppHost/appsettings.Development.json @@ -1,4 +1,11 @@ { + "Elasticsearch": { + "Port": 9215, + "ContainerName": "Exceptionless-Elasticsearch-9-5-Lookup", + "DataVolume": "exceptionless.elasticsearch-9-5-lookup.data.v1", + "KibanaPort": 5615, + "KibanaContainerName": "Exceptionless-Kibana-9-5-Lookup" + }, "Logging": { "LogLevel": { "Default": "Information", diff --git a/src/Exceptionless.Core/Bootstrapper.cs b/src/Exceptionless.Core/Bootstrapper.cs index 805a2f76d4..812476998c 100644 --- a/src/Exceptionless.Core/Bootstrapper.cs +++ b/src/Exceptionless.Core/Bootstrapper.cs @@ -77,6 +77,7 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO services.AddSingleton(); services.AddSingleton(s => s.GetRequiredService().Client); + services.AddSingleton(); services.AddSingleton(s => s.GetRequiredService()); services.AddStartupAction(); diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs index 02b15bdd89..76c84023ba 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs @@ -17,7 +17,7 @@ public sealed class StackIndex : VersionedIndex private readonly ExceptionlessElasticConfiguration _configuration; - public StackIndex(ExceptionlessElasticConfiguration configuration) : base(configuration, configuration.Options.ScopePrefix + "stacks", 1) + public StackIndex(ExceptionlessElasticConfiguration configuration) : base(configuration, configuration.Options.ScopePrefix + "stacks", 2) { _configuration = configuration; } @@ -27,11 +27,28 @@ public override void ConfigureIndex(CreateIndexRequestDescriptor idx) base.ConfigureIndex(idx); idx.Settings(s => s .Analysis(a => BuildAnalysis(a)) - .NumberOfShards(_configuration.Options.NumberOfShards) + .Mode("lookup") + .NumberOfShards(1) .NumberOfReplicas(_configuration.Options.NumberOfReplicas) .Priority(5)); } + protected override Task UpdateIndexAsync(string name, Action? descriptor = null) + { + if (descriptor is not null) + return base.UpdateIndexAsync(name, descriptor); + + // index.mode is a final creation-only setting. Foundatio derives updates from + // ConfigureIndex, so keep the mutable settings explicit for existing indexes. + return base.UpdateIndexAsync(name, update => update + .Reopen(true) + .Settings(new IndexSettings + { + NumberOfReplicas = _configuration.Options.NumberOfReplicas, + Priority = 5 + })); + } + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { map diff --git a/src/Exceptionless.Core/Repositories/Queries/Visitors/EventStackFilterQueryVisitor.cs b/src/Exceptionless.Core/Repositories/Queries/Visitors/EventStackFilterQueryVisitor.cs index 7d06961364..a3d5e4299a 100644 --- a/src/Exceptionless.Core/Repositories/Queries/Visitors/EventStackFilterQueryVisitor.cs +++ b/src/Exceptionless.Core/Repositories/Queries/Visitors/EventStackFilterQueryVisitor.cs @@ -65,7 +65,7 @@ public EventStackFilter() _stackQueryVisitor.AddVisitor(new RemoveFieldsQueryVisitor(f => !stackFields.Contains(f))); _stackQueryVisitor.AddVisitor(new CleanupQueryVisitor()); // handles stack special fields and changing event field names to their stack equivalent - _stackQueryVisitor.AddVisitor(new StackFilterQueryVisitor()); + _stackQueryVisitor.AddVisitor(new StackFilterQueryVisitor(_stackOnlyFields.Union(_stackOnlySpecialFields))); _stackQueryVisitor.AddVisitor(new CleanupQueryVisitor()); _invertedStackQueryVisitor = new ChainedQueryVisitor(); @@ -73,7 +73,7 @@ public EventStackFilter() _invertedStackQueryVisitor.AddVisitor(new RemoveFieldsQueryVisitor(f => !stackFields.Contains(f))); _invertedStackQueryVisitor.AddVisitor(new CleanupQueryVisitor()); // handles stack special fields and changing event field names to their stack equivalent - _invertedStackQueryVisitor.AddVisitor(new StackFilterQueryVisitor()); + _invertedStackQueryVisitor.AddVisitor(new StackFilterQueryVisitor(_stackOnlyFields.Union(_stackOnlySpecialFields))); _invertedStackQueryVisitor.AddVisitor(new CleanupQueryVisitor()); // inverts the filter _invertedStackQueryVisitor.AddVisitor(new InvertQueryVisitor(_stackNonInvertedFields)); @@ -109,13 +109,21 @@ public EventStackFilter() InvertedFilter = invertedResult?.ToString(), HasStatus = context.GetBoolean(nameof(StackFilter.HasStatus)), HasStackIds = context.GetBoolean(nameof(StackFilter.HasStackIds)), - HasStatusOpen = context.GetBoolean(nameof(StackFilter.HasStatusOpen)) + HasStatusOpen = context.GetBoolean(nameof(StackFilter.HasStatusOpen)), + HasStackOnlyCriteria = context.GetBoolean(nameof(StackFilter.HasStackOnlyCriteria)) }; } } public class StackFilterQueryVisitor : ChainableQueryVisitor { + private readonly ISet _stackOnlyFields; + + public StackFilterQueryVisitor(IEnumerable? stackOnlyFields = null) + { + _stackOnlyFields = new HashSet(stackOnlyFields ?? [], StringComparer.OrdinalIgnoreCase); + } + public override Task VisitAsync(TermNode node, IQueryVisitorContext context) { IQueryNode result = node; @@ -127,6 +135,9 @@ public class StackFilterQueryVisitor : ChainableQueryVisitor return Task.FromResult(null); } + if (_stackOnlyFields.Contains(node.Field)) + context.SetValue(nameof(StackFilter.HasStackOnlyCriteria), true); + // process special stack fields switch (node.Field?.ToLowerInvariant()) { @@ -216,4 +227,5 @@ public record StackFilter public required bool HasStatus { get; set; } public required bool HasStatusOpen { get; set; } public required bool HasStackIds { get; set; } + public required bool HasStackOnlyCriteria { get; set; } } diff --git a/src/Exceptionless.Core/Services/StackRollupSearchService.cs b/src/Exceptionless.Core/Services/StackRollupSearchService.cs new file mode 100644 index 0000000000..c2e67b0ab7 --- /dev/null +++ b/src/Exceptionless.Core/Services/StackRollupSearchService.cs @@ -0,0 +1,1084 @@ +using System.Diagnostics; +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Esql; +using Elastic.Clients.Elasticsearch.QueryDsl; +using Exceptionless.Core.Configuration; +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.Data; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Repositories.Configuration; +using Exceptionless.Core.Repositories.Queries; +using Foundatio.Repositories; +using Foundatio.Repositories.Elasticsearch.Queries.Builders; +using Foundatio.Repositories.Models; +using Foundatio.Repositories.Options; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Services; + +public interface IStackRollupSearchService +{ + Task RequiresLookupJoinAsync(string? filter, CancellationToken cancellationToken = default); + Task SearchEventsAsync(EventLookupSearchRequest request, CancellationToken cancellationToken = default); + Task CountEventsAsync(EventLookupCountRequest request, CancellationToken cancellationToken = default); + Task SearchAsync(StackRollupSearchRequest request, CancellationToken cancellationToken = default); + Task GetStatsAsync(StackRollupStatsRequest request, CancellationToken cancellationToken = default); + Task> GetProjectUserCountsAsync(StackRollupProjectUsersRequest request, CancellationToken cancellationToken = default); +} + +public sealed record EventLookupSearchRequest( + AppFilter? AppFilter, + DateTime UtcStart, + DateTime UtcEnd, + string? TimeExpression, + string? Filter, + string? Sort, + int Limit, + string? Before, + string? After, + bool IncludeTotal); + +public sealed record EventLookupSearchResult( + IReadOnlyCollection EventIds, + bool HasMore, + long? Total, + string? Before, + string? After); + +public sealed record EventLookupCountRequest( + AppFilter? AppFilter, + DateTime UtcStart, + DateTime UtcEnd, + TimeSpan Offset, + string? Filter, + string? Aggregations, + int BucketCount = 50); + +public sealed record StackRollupSearchRequest( + AppFilter? AppFilter, + DateTime UtcStart, + DateTime UtcEnd, + TimeSpan Offset, + string? TimeExpression, + string? Filter, + string? Sort, + int Limit, + string? Before, + string? After, + bool IncludeTotal); + +public sealed record StackRollupSearchResult( + IReadOnlyCollection Rows, + bool HasMore, + long? Total, + string? Before, + string? After); + +public sealed record StackRollupStatsRequest( + AppFilter? AppFilter, + DateTime UtcStart, + DateTime UtcEnd, + TimeSpan Offset, + string? Filter, + int BucketCount = 50); + +public sealed record StackRollupStatsResult( + long TotalEvents, + long TotalStacks, + long NewStacks, + IReadOnlyCollection Buckets, + long Documents = 0); + +public sealed record StackRollupStatsBucket(DateTime Date, long Events, long Stacks, long Documents = 0); + +public sealed record StackRollupProjectUsersRequest( + AppFilter AppFilter, + DateTime UtcStart, + DateTime UtcEnd, + IReadOnlyCollection ProjectIds); + +public sealed record StackRollupRow( + string StackId, + long Total, + long Users, + DateTime FirstOccurrence, + DateTime LastOccurrence); + +public sealed class InvalidStackRollupCursorException(string message) : Exception(message); +public sealed class InvalidEventLookupCursorException(string message) : Exception(message); + +public sealed class StackRollupSearchService : IStackRollupSearchService +{ + private const int CursorVersion = 1; + private static readonly TimeSpan ReadinessCacheDuration = TimeSpan.FromMinutes(1); + private readonly ElasticsearchClient _client; + private readonly ExceptionlessElasticConfiguration _configuration; + private readonly TimeProvider _timeProvider; + private readonly JsonSerializerOptions _serializerOptions; + private readonly EventStackFilter _eventStackFilter = new(); + private readonly ILogger _logger; + private readonly SemaphoreSlim _readinessLock = new(1, 1); + private StackRollupReadiness? _readiness; + private DateTimeOffset _readinessExpiresUtc; + + public StackRollupSearchService( + ElasticsearchClient client, + ExceptionlessElasticConfiguration configuration, + TimeProvider timeProvider, + JsonSerializerOptions serializerOptions, + ILoggerFactory loggerFactory) + { + _client = client; + _configuration = configuration; + _timeProvider = timeProvider; + _serializerOptions = serializerOptions; + _logger = loggerFactory.CreateLogger(); + } + + public async Task RequiresLookupJoinAsync(string? filter, CancellationToken cancellationToken = default) + { + if (String.IsNullOrWhiteSpace(filter)) + return false; + + var stackFilter = await _eventStackFilter.GetStackFilterAsync(StripAlternateInversion(filter)); + return stackFilter?.HasStackOnlyCriteria == true; + } + + public async Task SearchEventsAsync(EventLookupSearchRequest request, CancellationToken cancellationToken = default) + { + var sort = GetEventSort(request.Sort); + await EnsureReadyAsync(cancellationToken); + + string fingerprint = CreateEventFingerprint(request, sort.Value); + EventLookupCursor? cursor = DecodeEventCursor(request.Before ?? request.After, sort.Value, fingerprint); + DateTime utcStart = cursor is null ? request.UtcStart : new DateTime(cursor.UtcStart, DateTimeKind.Utc); + DateTime utcEnd = cursor is null ? request.UtcEnd : new DateTime(cursor.UtcEnd, DateTimeKind.Utc); + string normalizedFilter = StripAlternateInversion(request.Filter); + string? eventFilter = await _eventStackFilter.GetEventFilterAsync(normalizedFilter); + string? stackFilter = (await _eventStackFilter.GetStackFilterAsync(normalizedFilter))?.Filter; + Query? sourceFilter = await BuildSourceFilterAsync(request.AppFilter, utcStart, utcEnd, eventFilter); + var parameters = new List>>(); + string query = BuildEventQuery(request, sort, cursor, stackFilter, parameters); + var rows = await ExecuteEventRowsAsync(query, sourceFilter, parameters, cancellationToken); + long? total = request.IncludeTotal ? rows.FirstOrDefault()?.TotalEvents : null; + if (request.IncludeTotal && total is null) + total = await ExecuteJoinedEventTotalAsync(stackFilter, sourceFilter, cancellationToken); + + bool isBefore = request.Before is not null; + bool hasExtra = rows.Count > request.Limit; + if (hasExtra) + rows.RemoveAt(rows.Count - 1); + if (isBefore) + rows.Reverse(); + + bool hasPrevious = rows.Count > 0 && (isBefore ? hasExtra : request.After is not null); + bool hasNext = rows.Count > 0 && (isBefore || hasExtra); + return new EventLookupSearchResult( + rows.Select(row => row.EventId).ToArray(), + hasNext, + total, + hasPrevious ? EncodeEventCursor(rows[0], sort, utcStart, utcEnd, fingerprint) : null, + hasNext ? EncodeEventCursor(rows[^1], sort, utcStart, utcEnd, fingerprint) : null); + } + + public async Task CountEventsAsync(EventLookupCountRequest request, CancellationToken cancellationToken = default) + { + await EnsureReadyAsync(cancellationToken); + string normalizedFilter = StripAlternateInversion(request.Filter); + string? eventFilter = await _eventStackFilter.GetEventFilterAsync(normalizedFilter); + string? stackFilter = (await _eventStackFilter.GetStackFilterAsync(normalizedFilter))?.Filter; + Query? sourceFilter = await BuildSourceFilterAsync(request.AppFilter, request.UtcStart, request.UtcEnd, eventFilter); + + if (String.IsNullOrWhiteSpace(request.Aggregations)) + return new CountResult(await ExecuteJoinedEventTotalAsync(stackFilter, sourceFilter, cancellationToken)); + + if (IsTermsTagsAggregation(request.Aggregations)) + return await CountEventTagsAsync(stackFilter, sourceFilter, cancellationToken); + + if (!IsDashboardAggregation(request.Aggregations)) + throw new NotSupportedException("Stack filters currently support the event dashboard aggregations or terms:tags."); + + var stats = await GetStatsAsync(new StackRollupStatsRequest( + request.AppFilter, + request.UtcStart, + request.UtcEnd, + request.Offset, + request.Filter, + request.BucketCount), cancellationToken); + return ToCountResult(stats); + } + + public async Task SearchAsync(StackRollupSearchRequest request, CancellationToken cancellationToken = default) + { + var sort = GetSort(request.Sort); + await EnsureReadyAsync(cancellationToken); + + string fingerprint = CreateFingerprint(request); + StackRollupCursor? cursor = DecodeCursor(request.Before ?? request.After, sort.Value, fingerprint); + DateTime utcStart = cursor is null ? request.UtcStart : new DateTime(cursor.UtcStart, DateTimeKind.Utc); + DateTime utcEnd = cursor is null ? request.UtcEnd : new DateTime(cursor.UtcEnd, DateTimeKind.Utc); + string normalizedFilter = StripAlternateInversion(request.Filter); + string? eventFilter = await _eventStackFilter.GetEventFilterAsync(normalizedFilter); + string? stackFilter = (await _eventStackFilter.GetStackFilterAsync(normalizedFilter))?.Filter; + Query? sourceFilter = await BuildSourceFilterAsync(request.AppFilter, utcStart, utcEnd, eventFilter); + + var stopwatch = Stopwatch.StartNew(); + _logger.LogDebug( + "Executing ES|QL stack rollup sort {Sort}, direction {Direction}, range {UtcStart:o} to {UtcEnd:o}, event filter {HasEventFilter}, stack filter {HasStackFilter}", + sort.Value, + request.Before is not null ? "before" : request.After is not null ? "after" : "initial", + utcStart, + utcEnd, + !String.IsNullOrWhiteSpace(eventFilter), + !String.IsNullOrWhiteSpace(stackFilter)); + + var parameters = new List>>(); + string query = BuildQuery(request, sort, cursor, stackFilter, parameters, countOnly: false); + var rows = await ExecuteRowsAsync(query, sourceFilter, parameters, cancellationToken); + long? total = request.IncludeTotal ? rows.FirstOrDefault()?.TotalStacks : null; + + if (request.IncludeTotal && total is null) + { + parameters.Clear(); + string countQuery = BuildQuery(request, sort, cursor: null, stackFilter, parameters, countOnly: true); + total = await ExecuteTotalAsync(countQuery, sourceFilter, parameters, cancellationToken); + } + + bool isBefore = request.Before is not null; + bool hasExtra = rows.Count > request.Limit; + if (hasExtra) + rows.RemoveAt(rows.Count - 1); + + if (isBefore) + rows.Reverse(); + + bool hasPrevious = rows.Count > 0 && (isBefore ? hasExtra : request.After is not null); + bool hasNext = rows.Count > 0 && (isBefore || hasExtra); + string? before = hasPrevious ? EncodeCursor(rows[0], sort, utcStart, utcEnd, fingerprint) : null; + string? after = hasNext ? EncodeCursor(rows[^1], sort, utcStart, utcEnd, fingerprint) : null; + + _logger.LogDebug( + "Completed ES|QL stack rollup sort {Sort}, direction {Direction}, rows {RowCount}, has more {HasMore}, duration {DurationMs}ms", + sort.Value, + isBefore ? "before" : request.After is not null ? "after" : "initial", + rows.Count, + hasNext, + stopwatch.Elapsed.TotalMilliseconds); + + return new StackRollupSearchResult( + rows.Select(ToPublicRow).ToArray(), + hasNext, + total, + before, + after); + } + + public async Task GetStatsAsync(StackRollupStatsRequest request, CancellationToken cancellationToken = default) + { + await EnsureReadyAsync(cancellationToken); + string normalizedFilter = StripAlternateInversion(request.Filter); + string? eventFilter = await _eventStackFilter.GetEventFilterAsync(normalizedFilter); + string? stackFilter = (await _eventStackFilter.GetStackFilterAsync(normalizedFilter))?.Filter; + Query? sourceFilter = await BuildSourceFilterAsync(request.AppFilter, request.UtcStart, request.UtcEnd, eventFilter); + var parameters = new List>>(); + string query = BuildStatsQuery(request, stackFilter, parameters); + + using var response = await _client.Esql.QueryAsync(new EsqlQueryRequest(query) + { + AllowPartialResults = false, + Columnar = false, + Filter = sourceFilter, + Format = EsqlFormat.Json, + Params = new Union>, ICollection>>>(parameters) + }, cancellationToken); + + if (!response.IsValidResponse) + { + _logger.LogWarning("Stack rollup stats lookup join failed with Elasticsearch status {StatusCode}", response.ApiCallDetails?.HttpStatusCode); + throw new ApplicationException("The stack rollup stats query failed."); + } + + using var document = await JsonDocument.ParseAsync(response.Body, cancellationToken: cancellationToken); + return ReadStats(document.RootElement); + } + + public async Task> GetProjectUserCountsAsync(StackRollupProjectUsersRequest request, CancellationToken cancellationToken = default) + { + if (request.ProjectIds.Count == 0) + return new Dictionary(); + + await EnsureReadyAsync(cancellationToken); + Query? sourceFilter = await BuildSourceFilterAsync(request.AppFilter, request.UtcStart, request.UtcEnd, eventFilter: null); + string eventIndex = ValidateIndexName(_configuration.Events.Name); + string stackIndex = ValidateIndexName(_configuration.Stacks.Name); + string userField = EscapeIdentifier(EventIndexExtensions.DataPath(Event.KnownDataKeys.UserInfo, user => user.Identity) + ".keyword"); + var parameters = new List>>(); + AddParameter(parameters, "project_ids", request.ProjectIds.Select(FieldValue.String).ToArray()); + string query = new StringBuilder() + .Append("FROM ").Append(eventIndex) + .Append(" | KEEP stack_id, project_id, ").Append(userField) + .Append(" | RENAME project_id AS event_project_id, ").Append(userField).Append(" AS event_user") + .Append(" | LOOKUP JOIN ").Append(stackIndex).Append(" ON stack_id == id AND is_deleted == false") + .Append(" | WHERE id IS NOT NULL AND event_project_id IN (?project_ids)") + .Append(" | STATS users = COUNT_DISTINCT(event_user) BY project_id = event_project_id") + .Append(" | KEEP project_id, users") + .ToString(); + + using var response = await _client.Esql.QueryAsync(new EsqlQueryRequest(query) + { + AllowPartialResults = false, + Columnar = false, + Filter = sourceFilter, + Format = EsqlFormat.Json, + Params = new Union>, ICollection>>>(parameters) + }, cancellationToken); + + if (!response.IsValidResponse) + { + _logger.LogWarning("Stack rollup project user lookup join failed with Elasticsearch status {StatusCode}", response.ApiCallDetails?.HttpStatusCode); + throw new ApplicationException("The stack rollup project user query failed."); + } + + using var document = await JsonDocument.ParseAsync(response.Body, cancellationToken: cancellationToken); + return ReadProjectUserCounts(document.RootElement); + } + + private async Task EnsureReadyAsync(CancellationToken cancellationToken) + { + var readiness = await GetReadinessAsync(cancellationToken); + if (readiness.IsReady) + return; + + _logger.LogError("Stack rollup lookup join prerequisite failed: {Reason}", readiness.Reason); + throw new InvalidOperationException($"The stack rollup lookup join prerequisite failed: {readiness.Reason}."); + } + + private async Task BuildSourceFilterAsync(AppFilter? appFilter, DateTime utcStart, DateTime utcEnd, string? eventFilter) + { + var query = new RepositoryQuery() + .AppFilter(appFilter) + .DateRange(utcStart, utcEnd, (PersistentEvent e) => e.Date) + .FilterExpression(eventFilter); + + var options = new CommandOptions() + .TimeProvider(_timeProvider) + .ElasticIndex(_configuration.Events) + .DocumentType(typeof(PersistentEvent)); + var context = new QueryBuilderContext(query, options); + await _configuration.Events.QueryBuilder.BuildAsync(context); + return context.Filter; + } + + private string BuildEventQuery( + EventLookupSearchRequest request, + EventLookupSort sort, + EventLookupCursor? cursor, + string? stackFilter, + ICollection>> parameters) + { + string eventIndex = ValidateIndexName(_configuration.Events.Name); + string stackIndex = ValidateIndexName(_configuration.Stacks.Name); + var query = new StringBuilder() + .Append("FROM ").Append(eventIndex).Append(" METADATA _id") + .Append(" | KEEP _id, stack_id, date") + .Append(" | LOOKUP JOIN ").Append(stackIndex) + .Append(" ON stack_id == id AND is_deleted == false"); + + if (!String.IsNullOrWhiteSpace(stackFilter)) + { + query.Append(" AND QSTR(?stack_filter, {\"default_operator\": \"AND\"})"); + AddParameter(parameters, "stack_filter", FieldValue.String(stackFilter)); + } + + query.Append(" | WHERE id IS NOT NULL"); + if (request.IncludeTotal) + query.Append(" | INLINE STATS total_events = COUNT(*)"); + + bool isBefore = request.Before is not null; + if (cursor is not null) + { + string primaryComparison = isBefore + ? sort.Ascending ? "<" : ">" + : sort.Ascending ? ">" : "<"; + string idComparison = isBefore ? "<" : ">"; + query + .Append(" | WHERE date ").Append(primaryComparison).Append(" TO_DATETIME(?cursor_date)") + .Append(" OR (date == TO_DATETIME(?cursor_date) AND _id ").Append(idComparison).Append(" ?cursor_event_id)"); + AddParameter(parameters, "cursor_date", FieldValue.String(new DateTime(cursor.Date, DateTimeKind.Utc).ToString("O", CultureInfo.InvariantCulture))); + AddParameter(parameters, "cursor_event_id", FieldValue.String(cursor.EventId)); + } + + bool queryAscending = isBefore ? !sort.Ascending : sort.Ascending; + query + .Append(" | SORT date ").Append(queryAscending ? "ASC" : "DESC") + .Append(", _id ").Append(isBefore ? "DESC" : "ASC") + .Append(" | LIMIT ").Append(request.Limit + 1) + .Append(" | KEEP _id, date"); + if (request.IncludeTotal) + query.Append(", total_events"); + + return query.ToString(); + } + + private async Task> ExecuteEventRowsAsync( + string query, + Query? sourceFilter, + ICollection>> parameters, + CancellationToken cancellationToken) + { + using var response = await _client.Esql.QueryAsync(new EsqlQueryRequest(query) + { + AllowPartialResults = false, + Columnar = false, + Filter = sourceFilter, + Format = EsqlFormat.Json, + Params = new Union>, ICollection>>>(parameters) + }, cancellationToken); + + if (!response.IsValidResponse) + { + _logger.LogWarning("Event lookup join failed with Elasticsearch status {StatusCode}", response.ApiCallDetails?.HttpStatusCode); + throw new ApplicationException("The event lookup join query failed."); + } + + using var document = await JsonDocument.ParseAsync(response.Body, cancellationToken: cancellationToken); + if (!document.RootElement.TryGetProperty("columns", out var columns) || !document.RootElement.TryGetProperty("values", out var values)) + throw new JsonException("The ES|QL event lookup response did not contain columns and values."); + + var columnIndexes = GetColumnIndexes(columns); + int idIndex = columnIndexes["_id"]; + int dateIndex = columnIndexes["date"]; + int? totalIndex = columnIndexes.TryGetValue("total_events", out int index) ? index : null; + return values.EnumerateArray() + .Select(value => new EventLookupEsqlRow( + value[idIndex].GetString() ?? throw new JsonException("An event lookup row did not contain an event id."), + value[dateIndex].GetDateTime().ToUniversalTime(), + totalIndex.HasValue ? value[totalIndex.Value].GetInt64() : null)) + .ToList(); + } + + private Task ExecuteJoinedEventTotalAsync(string? stackFilter, Query? sourceFilter, CancellationToken cancellationToken) + { + string eventIndex = ValidateIndexName(_configuration.Events.Name); + string stackIndex = ValidateIndexName(_configuration.Stacks.Name); + var parameters = new List>>(); + var query = new StringBuilder() + .Append("FROM ").Append(eventIndex) + .Append(" | KEEP stack_id") + .Append(" | LOOKUP JOIN ").Append(stackIndex) + .Append(" ON stack_id == id AND is_deleted == false"); + if (!String.IsNullOrWhiteSpace(stackFilter)) + { + query.Append(" AND QSTR(?stack_filter, {\"default_operator\": \"AND\"})"); + AddParameter(parameters, "stack_filter", FieldValue.String(stackFilter)); + } + query.Append(" | WHERE id IS NOT NULL | STATS total_events = COUNT(*) | KEEP total_events"); + return ExecuteTotalAsync(query.ToString(), sourceFilter, parameters, cancellationToken); + } + + private async Task CountEventTagsAsync(string? stackFilter, Query? sourceFilter, CancellationToken cancellationToken) + { + string eventIndex = ValidateIndexName(_configuration.Events.Name); + string stackIndex = ValidateIndexName(_configuration.Stacks.Name); + var parameters = new List>>(); + var query = new StringBuilder() + .Append("FROM ").Append(eventIndex) + .Append(" | KEEP stack_id, tags") + .Append(" | LOOKUP JOIN ").Append(stackIndex) + .Append(" ON stack_id == id AND is_deleted == false"); + if (!String.IsNullOrWhiteSpace(stackFilter)) + { + query.Append(" AND QSTR(?stack_filter, {\"default_operator\": \"AND\"})"); + AddParameter(parameters, "stack_filter", FieldValue.String(stackFilter)); + } + query + .Append(" | WHERE id IS NOT NULL") + .Append(" | MV_EXPAND tags") + .Append(" | STATS tag_total = COUNT(*) BY tag = tags") + .Append(" | SORT tag_total DESC, tag ASC | LIMIT 1000 | KEEP tag, tag_total"); + + using var response = await _client.Esql.QueryAsync(new EsqlQueryRequest(query.ToString()) + { + AllowPartialResults = false, + Columnar = false, + Filter = sourceFilter, + Format = EsqlFormat.Json, + Params = new Union>, ICollection>>>(parameters) + }, cancellationToken); + if (!response.IsValidResponse) + throw new ApplicationException("The event tag lookup join query failed."); + + using var document = await JsonDocument.ParseAsync(response.Body, cancellationToken: cancellationToken); + if (!document.RootElement.TryGetProperty("columns", out var columns) || !document.RootElement.TryGetProperty("values", out var values)) + throw new JsonException("The ES|QL event tag response did not contain columns and values."); + var columnIndexes = GetColumnIndexes(columns); + int tagIndex = columnIndexes["tag"]; + int tagTotalIndex = columnIndexes["tag_total"]; + var buckets = new List(); + foreach (var value in values.EnumerateArray()) + { + buckets.Add(new KeyedBucket(null) + { + Key = value[tagIndex].GetString() ?? String.Empty, + Total = value[tagTotalIndex].GetInt64(), + Data = new Dictionary { ["@type"] = "string" } + }); + } + + long total = await ExecuteJoinedEventTotalAsync(stackFilter, sourceFilter, cancellationToken); + return new CountResult(total, new Dictionary + { + ["terms_tags"] = new BucketAggregate + { + Items = buckets, + Data = new Dictionary { ["@type"] = "bucket" } + } + }); + } + + private string BuildQuery( + StackRollupSearchRequest request, + StackRollupSort sort, + StackRollupCursor? cursor, + string? stackFilter, + ICollection>> parameters, + bool countOnly) + { + string eventIndex = ValidateIndexName(_configuration.Events.Name); + string stackIndex = ValidateIndexName(_configuration.Stacks.Name); + string userField = EscapeIdentifier(EventIndexExtensions.DataPath(Event.KnownDataKeys.UserInfo, user => user.Identity) + ".keyword"); + var query = new StringBuilder() + .Append("FROM ").Append(eventIndex) + .Append(" | KEEP stack_id, count, date, ").Append(userField) + .Append(" | RENAME ").Append(userField).Append(" AS event_user") + .Append(" | LOOKUP JOIN ").Append(stackIndex) + .Append(" ON stack_id == id AND is_deleted == false"); + + if (!String.IsNullOrWhiteSpace(stackFilter)) + { + query.Append(" AND QSTR(?stack_filter, {\"default_operator\": \"AND\"})"); + AddParameter(parameters, "stack_filter", FieldValue.String(stackFilter)); + } + + query + .Append(" | WHERE id IS NOT NULL") + .Append(" | STATS event_total = SUM(COALESCE(count, 1)), event_users = COUNT_DISTINCT(event_user), event_first = MIN(date), event_last = MAX(date) BY stack_id"); + + if (countOnly) + return query.Append(" | STATS total_stacks = COUNT(*) | KEEP total_stacks").ToString(); + + if (request.IncludeTotal) + query.Append(" | INLINE STATS total_stacks = COUNT(*)"); + + bool isBefore = request.Before is not null; + if (cursor is not null) + { + string primaryComparison = isBefore + ? sort.Ascending ? "<" : ">" + : sort.Ascending ? ">" : "<"; + string idComparison = isBefore ? "<" : ">"; + string metricParameter = sort.IsDate ? "TO_DATETIME(?cursor_metric)" : "?cursor_metric"; + query + .Append(" | WHERE ").Append(sort.Metric).Append(' ').Append(primaryComparison).Append(' ').Append(metricParameter) + .Append(" OR (").Append(sort.Metric).Append(" == ").Append(metricParameter) + .Append(" AND stack_id ").Append(idComparison).Append(" ?cursor_stack_id)"); + AddParameter(parameters, "cursor_metric", sort.IsDate + ? FieldValue.String(new DateTime(cursor.Metric, DateTimeKind.Utc).ToString("O", CultureInfo.InvariantCulture)) + : FieldValue.Long(cursor.Metric)); + AddParameter(parameters, "cursor_stack_id", FieldValue.String(cursor.StackId)); + } + + bool queryAscending = isBefore ? !sort.Ascending : sort.Ascending; + string primarySort = queryAscending ? "ASC" : "DESC"; + string idSort = isBefore ? "DESC" : "ASC"; + query + .Append(" | SORT ").Append(sort.Metric).Append(' ').Append(primarySort).Append(", stack_id ").Append(idSort) + .Append(" | LIMIT ").Append(request.Limit + 1) + .Append(" | KEEP stack_id, event_total, event_users, event_first, event_last"); + + if (request.IncludeTotal) + query.Append(", total_stacks"); + + return query.ToString(); + } + + private string BuildStatsQuery( + StackRollupStatsRequest request, + string? stackFilter, + ICollection>> parameters) + { + string eventIndex = ValidateIndexName(_configuration.Events.Name); + string stackIndex = ValidateIndexName(_configuration.Stacks.Name); + int bucketCount = Math.Clamp(request.BucketCount, 1, 100); + string timeZone = FormatTimeZone(request.Offset); + string utcStart = request.UtcStart.ToString("O", CultureInfo.InvariantCulture); + string utcEnd = request.UtcEnd.ToString("O", CultureInfo.InvariantCulture); + var query = new StringBuilder() + .Append("SET time_zone = \"").Append(timeZone).Append("\"; FROM ").Append(eventIndex) + .Append(" | KEEP stack_id, count, date, is_first_occurrence") + .Append(" | LOOKUP JOIN ").Append(stackIndex) + .Append(" ON stack_id == id AND is_deleted == false"); + + if (!String.IsNullOrWhiteSpace(stackFilter)) + { + query.Append(" AND QSTR(?stack_filter, {\"default_operator\": \"AND\"})"); + AddParameter(parameters, "stack_filter", FieldValue.String(stackFilter)); + } + + return query + .Append(" | WHERE id IS NOT NULL") + .Append(" | INLINE STATS total_documents = COUNT(*), total_events = SUM(COALESCE(count, 1)), total_stacks = COUNT_DISTINCT(stack_id, 40000), new_stacks = SUM(CASE(is_first_occurrence, 1, 0))") + .Append(" | STATS documents = COUNT(*), events = SUM(COALESCE(count, 1)), stacks = COUNT_DISTINCT(stack_id), total_documents = MAX(total_documents), total_events = MAX(total_events), total_stacks = MAX(total_stacks), new_stacks = MAX(new_stacks)") + .Append(" BY bucket = BUCKET(date, ").Append(bucketCount).Append(", \"").Append(utcStart).Append("\", \"").Append(utcEnd).Append("\")") + .Append(" | SORT bucket | LIMIT ").Append(bucketCount + 2) + .Append(" | KEEP bucket, documents, events, stacks, total_documents, total_events, total_stacks, new_stacks") + .ToString(); + } + + private async Task> ExecuteRowsAsync( + string query, + Query? sourceFilter, + ICollection>> parameters, + CancellationToken cancellationToken) + { + using var response = await _client.Esql.QueryAsync(new EsqlQueryRequest(query) + { + AllowPartialResults = false, + Columnar = false, + Filter = sourceFilter, + Format = EsqlFormat.Json, + Params = new Union>, ICollection>>>(parameters) + }, cancellationToken); + + if (!response.IsValidResponse) + { + _logger.LogWarning("Stack rollup lookup join failed with Elasticsearch status {StatusCode}", response.ApiCallDetails?.HttpStatusCode); + throw new ApplicationException("The experimental stack rollup query failed."); + } + + using var document = await JsonDocument.ParseAsync(response.Body, cancellationToken: cancellationToken); + return ReadRows(document.RootElement); + } + + private async Task ExecuteTotalAsync( + string query, + Query? sourceFilter, + ICollection>> parameters, + CancellationToken cancellationToken) + { + using var response = await _client.Esql.QueryAsync(new EsqlQueryRequest(query) + { + AllowPartialResults = false, + Columnar = false, + Filter = sourceFilter, + Format = EsqlFormat.Json, + Params = new Union>, ICollection>>>(parameters) + }, cancellationToken); + + if (!response.IsValidResponse) + throw new ApplicationException("The experimental stack rollup count query failed."); + + using var document = await JsonDocument.ParseAsync(response.Body, cancellationToken: cancellationToken); + if (!document.RootElement.TryGetProperty("values", out var values) || values.GetArrayLength() == 0) + return 0; + + return values[0][0].GetInt64(); + } + + private static List ReadRows(JsonElement root) + { + if (!root.TryGetProperty("columns", out var columns) || !root.TryGetProperty("values", out var values)) + throw new JsonException("The ES|QL stack rollup response did not contain columns and values."); + + var columnIndexes = columns.EnumerateArray() + .Select((column, index) => (Name: column.GetProperty("name").GetString(), Index: index)) + .Where(column => column.Name is not null) + .ToDictionary(column => column.Name!, column => column.Index, StringComparer.Ordinal); + + int stackIdIndex = columnIndexes["stack_id"]; + int totalIndex = columnIndexes["event_total"]; + int usersIndex = columnIndexes["event_users"]; + int firstIndex = columnIndexes["event_first"]; + int lastIndex = columnIndexes["event_last"]; + int? totalStacksIndex = columnIndexes.TryGetValue("total_stacks", out int index) ? index : null; + var rows = new List(); + foreach (var value in values.EnumerateArray()) + { + rows.Add(new StackRollupEsqlRow( + value[stackIdIndex].GetString() ?? throw new JsonException("A stack rollup row did not contain a stack id."), + value[totalIndex].GetInt64(), + value[usersIndex].GetInt64(), + value[firstIndex].GetDateTime().ToUniversalTime(), + value[lastIndex].GetDateTime().ToUniversalTime(), + totalStacksIndex.HasValue ? value[totalStacksIndex.Value].GetInt64() : null)); + } + + return rows; + } + + private static StackRollupStatsResult ReadStats(JsonElement root) + { + if (!root.TryGetProperty("columns", out var columns) || !root.TryGetProperty("values", out var values)) + throw new JsonException("The ES|QL stack rollup stats response did not contain columns and values."); + if (values.GetArrayLength() == 0) + return new StackRollupStatsResult(0, 0, 0, []); + + var columnIndexes = GetColumnIndexes(columns); + int bucketIndex = columnIndexes["bucket"]; + int documentsIndex = columnIndexes["documents"]; + int eventsIndex = columnIndexes["events"]; + int stacksIndex = columnIndexes["stacks"]; + int totalDocumentsIndex = columnIndexes["total_documents"]; + int totalEventsIndex = columnIndexes["total_events"]; + int totalStacksIndex = columnIndexes["total_stacks"]; + int newStacksIndex = columnIndexes["new_stacks"]; + var buckets = new List(values.GetArrayLength()); + long totalEvents = 0; + long totalStacks = 0; + long newStacks = 0; + long totalDocuments = 0; + foreach (var value in values.EnumerateArray()) + { + totalDocuments = value[totalDocumentsIndex].GetInt64(); + totalEvents = value[totalEventsIndex].GetInt64(); + totalStacks = value[totalStacksIndex].GetInt64(); + newStacks = value[newStacksIndex].GetInt64(); + buckets.Add(new StackRollupStatsBucket( + value[bucketIndex].GetDateTimeOffset().UtcDateTime, + value[eventsIndex].GetInt64(), + value[stacksIndex].GetInt64(), + value[documentsIndex].GetInt64())); + } + + return new StackRollupStatsResult(totalEvents, totalStacks, newStacks, buckets, totalDocuments); + } + + private static IReadOnlyDictionary ReadProjectUserCounts(JsonElement root) + { + if (!root.TryGetProperty("columns", out var columns) || !root.TryGetProperty("values", out var values)) + throw new JsonException("The ES|QL project user response did not contain columns and values."); + + var columnIndexes = GetColumnIndexes(columns); + int projectIndex = columnIndexes["project_id"]; + int usersIndex = columnIndexes["users"]; + var result = new Dictionary(StringComparer.Ordinal); + foreach (var value in values.EnumerateArray()) + { + string projectId = value[projectIndex].GetString() ?? throw new JsonException("A project user row did not contain a project id."); + result[projectId] = value[usersIndex].GetInt64(); + } + + return result; + } + + private static IReadOnlyDictionary GetColumnIndexes(JsonElement columns) + => columns.EnumerateArray() + .Select((column, index) => (Name: column.GetProperty("name").GetString(), Index: index)) + .Where(column => column.Name is not null) + .ToDictionary(column => column.Name!, column => column.Index, StringComparer.Ordinal); + + private static bool IsDashboardAggregation(string aggregations) + { + string value = aggregations.Replace(" ", String.Empty, StringComparison.Ordinal).ToLowerInvariant(); + return value.Contains("date:(date", StringComparison.Ordinal) + && value.Contains("cardinality:stack", StringComparison.Ordinal) + && value.Contains("sum:count~1", StringComparison.Ordinal) + && value.Contains("terms:(first@include:true)", StringComparison.Ordinal); + } + + private static bool IsTermsTagsAggregation(string aggregations) + => String.Equals(aggregations.Replace(" ", String.Empty, StringComparison.Ordinal), "terms:tags", StringComparison.OrdinalIgnoreCase); + + private static CountResult ToCountResult(StackRollupStatsResult stats) + { + static ValueAggregate Metric(double value) => new() + { + Value = value, + Data = new Dictionary { ["@type"] = "value" } + }; + + var dateBuckets = stats.Buckets.Select(bucket => (IBucket)new DateHistogramBucket(bucket.Date, new Dictionary + { + ["cardinality_stack"] = Metric(bucket.Stacks), + ["sum_count"] = Metric(bucket.Events) + }) + { + Key = new DateTimeOffset(bucket.Date).ToUnixTimeMilliseconds(), + KeyAsString = bucket.Date.ToString("O", CultureInfo.InvariantCulture), + Total = bucket.Documents, + Data = new Dictionary { ["@type"] = "datehistogram" } + }).ToArray(); + + var firstBuckets = new IBucket[] + { + new KeyedBucket(null) + { + Key = true, + Total = stats.NewStacks, + Data = new Dictionary { ["@type"] = "bool" } + } + }; + + return new CountResult(stats.Documents, new Dictionary + { + ["date_date"] = new BucketAggregate + { + Items = dateBuckets, + Data = new Dictionary { ["@type"] = "bucket" } + }, + ["cardinality_stack"] = Metric(stats.TotalStacks), + ["terms_first"] = new BucketAggregate + { + Items = firstBuckets, + Data = new Dictionary { ["@type"] = "bucket" } + }, + ["sum_count"] = Metric(stats.TotalEvents) + }); + } + + private async Task GetReadinessAsync(CancellationToken cancellationToken) + { + DateTimeOffset now = _timeProvider.GetUtcNow(); + if (_readiness is not null && now < _readinessExpiresUtc) + return _readiness; + + await _readinessLock.WaitAsync(cancellationToken); + try + { + now = _timeProvider.GetUtcNow(); + if (_readiness is not null && now < _readinessExpiresUtc) + return _readiness; + + _readiness = await CheckReadinessAsync(cancellationToken); + _readinessExpiresUtc = now.Add(ReadinessCacheDuration); + return _readiness; + } + finally + { + _readinessLock.Release(); + } + } + + private async Task CheckReadinessAsync(CancellationToken cancellationToken) + { + var info = await _client.InfoAsync(cancellationToken); + if (!info.IsValidResponse || !System.Version.TryParse(info.Version.Number, out var version) || version < new System.Version(9, 5)) + return new StackRollupReadiness(false, "elasticsearch-version"); + + var settings = await _client.Indices.GetSettingsAsync((Indices)_configuration.Stacks.Name, cancellationToken); + if (!settings.IsValidResponse || settings.Settings.Count != 1) + return new StackRollupReadiness(false, "stack-alias-target"); + + var indexSettings = settings.Settings.Single().Value.Settings?.Index; + if (indexSettings is null || !String.Equals(indexSettings.Mode, "lookup", StringComparison.OrdinalIgnoreCase)) + return new StackRollupReadiness(false, "stack-index-mode"); + + int shards = indexSettings.NumberOfShards is null + ? 0 + : indexSettings.NumberOfShards.Match(value => value, value => Int32.TryParse(value, out int parsed) ? parsed : 0); + return shards == 1 + ? new StackRollupReadiness(true, "ready") + : new StackRollupReadiness(false, "stack-primary-shards"); + } + + private string EncodeCursor(StackRollupEsqlRow row, StackRollupSort sort, DateTime utcStart, DateTime utcEnd, string fingerprint) + { + long metric = sort.Metric switch + { + "event_total" => row.Total, + "event_users" => row.Users, + "event_first" => row.FirstOccurrence.Ticks, + "event_last" => row.LastOccurrence.Ticks, + _ => throw new InvalidOperationException("Unsupported stack rollup metric.") + }; + var cursor = new StackRollupCursor(CursorVersion, sort.Value, metric, row.StackId, utcStart.Ticks, utcEnd.Ticks, fingerprint); + byte[] bytes = JsonSerializer.SerializeToUtf8Bytes(cursor, _serializerOptions); + return Convert.ToBase64String(bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_'); + } + + private string EncodeEventCursor(EventLookupEsqlRow row, EventLookupSort sort, DateTime utcStart, DateTime utcEnd, string fingerprint) + { + var cursor = new EventLookupCursor(CursorVersion, sort.Value, row.Date.Ticks, row.EventId, utcStart.Ticks, utcEnd.Ticks, fingerprint); + byte[] bytes = JsonSerializer.SerializeToUtf8Bytes(cursor, _serializerOptions); + return Convert.ToBase64String(bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_'); + } + + private EventLookupCursor? DecodeEventCursor(string? token, string sort, string fingerprint) + { + if (String.IsNullOrWhiteSpace(token)) + return null; + + try + { + string base64 = token.Replace('-', '+').Replace('_', '/'); + base64 = base64.PadRight(base64.Length + (4 - base64.Length % 4) % 4, '='); + var cursor = JsonSerializer.Deserialize(Convert.FromBase64String(base64), _serializerOptions); + if (cursor is null + || cursor.Version != CursorVersion + || !String.Equals(cursor.Sort, sort, StringComparison.Ordinal) + || !String.Equals(cursor.Fingerprint, fingerprint, StringComparison.Ordinal) + || String.IsNullOrWhiteSpace(cursor.EventId) + || cursor.Date < DateTime.MinValue.Ticks + || cursor.Date > DateTime.MaxValue.Ticks + || cursor.UtcStart < DateTime.MinValue.Ticks + || cursor.UtcStart > DateTime.MaxValue.Ticks + || cursor.UtcEnd < DateTime.MinValue.Ticks + || cursor.UtcEnd > DateTime.MaxValue.Ticks + || cursor.UtcStart > cursor.UtcEnd) + { + throw new InvalidEventLookupCursorException("The event pagination cursor is not valid for this query."); + } + + return cursor; + } + catch (InvalidEventLookupCursorException) + { + throw; + } + catch (Exception ex) when (ex is FormatException or JsonException or OverflowException) + { + throw new InvalidEventLookupCursorException("The event pagination cursor is malformed."); + } + } + + private StackRollupCursor? DecodeCursor(string? token, string sort, string fingerprint) + { + if (String.IsNullOrWhiteSpace(token)) + return null; + + try + { + string base64 = token.Replace('-', '+').Replace('_', '/'); + base64 = base64.PadRight(base64.Length + (4 - base64.Length % 4) % 4, '='); + var cursor = JsonSerializer.Deserialize(Convert.FromBase64String(base64), _serializerOptions); + if (cursor is null + || cursor.Version != CursorVersion + || !String.Equals(cursor.Sort, sort, StringComparison.Ordinal) + || !String.Equals(cursor.Fingerprint, fingerprint, StringComparison.Ordinal) + || String.IsNullOrWhiteSpace(cursor.StackId) + || cursor.UtcStart < DateTime.MinValue.Ticks + || cursor.UtcStart > DateTime.MaxValue.Ticks + || cursor.UtcEnd < DateTime.MinValue.Ticks + || cursor.UtcEnd > DateTime.MaxValue.Ticks + || cursor.UtcStart > cursor.UtcEnd + || GetSort(sort).IsDate && (cursor.Metric < DateTime.MinValue.Ticks || cursor.Metric > DateTime.MaxValue.Ticks)) + { + throw new InvalidStackRollupCursorException("The stack pagination cursor is not valid for this query."); + } + + return cursor; + } + catch (InvalidStackRollupCursorException) + { + throw; + } + catch (Exception ex) when (ex is FormatException or JsonException or OverflowException) + { + throw new InvalidStackRollupCursorException("The stack pagination cursor is malformed."); + } + } + + private static string CreateFingerprint(StackRollupSearchRequest request) + { + string organizations = String.Join(',', request.AppFilter?.Organizations.Select(organization => organization.Id).Order(StringComparer.Ordinal) ?? Enumerable.Empty()); + string projects = String.Join(',', request.AppFilter?.Projects?.Select(project => project.Id).Order(StringComparer.Ordinal) ?? Enumerable.Empty()); + string value = String.Join('\n', [ + GetSort(request.Sort).Value, + request.Filter ?? String.Empty, + request.TimeExpression ?? String.Empty, + request.Offset.Ticks.ToString(CultureInfo.InvariantCulture), + organizations, + projects, + request.AppFilter?.Stack?.Id ?? String.Empty + ]); + return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(value))); + } + + private static string CreateEventFingerprint(EventLookupSearchRequest request, string sort) + { + string organizations = String.Join(',', request.AppFilter?.Organizations.Select(organization => organization.Id).Order(StringComparer.Ordinal) ?? Enumerable.Empty()); + string projects = String.Join(',', request.AppFilter?.Projects?.Select(project => project.Id).Order(StringComparer.Ordinal) ?? Enumerable.Empty()); + string value = String.Join('\n', [ + sort, + request.Filter ?? String.Empty, + request.TimeExpression ?? String.Empty, + organizations, + projects, + request.AppFilter?.Stack?.Id ?? String.Empty + ]); + return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(value))); + } + + private static StackRollupRow ToPublicRow(StackRollupEsqlRow row) => new( + row.StackId, + row.Total, + row.Users, + row.FirstOccurrence, + row.LastOccurrence); + + private static StackRollupSort GetSort(string? sort) => (String.IsNullOrWhiteSpace(sort) ? "-total" : sort.Trim()) switch + { + "total" => new StackRollupSort("total", "event_total", false, true), + "-total" => new StackRollupSort("-total", "event_total", false, false), + "users" => new StackRollupSort("users", "event_users", false, true), + "-users" => new StackRollupSort("-users", "event_users", false, false), + "first_occurrence" => new StackRollupSort("first_occurrence", "event_first", true, true), + "-first_occurrence" => new StackRollupSort("-first_occurrence", "event_first", true, false), + "last_occurrence" => new StackRollupSort("last_occurrence", "event_last", true, true), + "-last_occurrence" => new StackRollupSort("-last_occurrence", "event_last", true, false), + _ => throw new ArgumentOutOfRangeException(nameof(sort), sort, "Unsupported stack rollup sort.") + }; + + private static EventLookupSort GetEventSort(string? sort) => (String.IsNullOrWhiteSpace(sort) ? "-date" : sort.Trim()) switch + { + "date" => new EventLookupSort("date", true), + "-date" => new EventLookupSort("-date", false), + _ => throw new ArgumentOutOfRangeException(nameof(sort), sort, "Unsupported event lookup sort.") + }; + + private static string StripAlternateInversion(string? filter) => filter?.StartsWith("@!", StringComparison.Ordinal) == true ? filter[2..] : filter ?? String.Empty; + + private static string ValidateIndexName(string index) + { + if (String.IsNullOrWhiteSpace(index) || index.Any(character => !Char.IsAsciiLetterOrDigit(character) && character is not '-' and not '_' and not '.')) + throw new InvalidOperationException("The configured Elasticsearch index alias cannot be used in ES|QL."); + + return index; + } + + private static string EscapeIdentifier(string field) => $"`{field.Replace("`", "``", StringComparison.Ordinal)}`"; + + private static string FormatTimeZone(TimeSpan offset) + { + if (offset < TimeSpan.FromHours(-14) || offset > TimeSpan.FromHours(14)) + throw new ArgumentOutOfRangeException(nameof(offset), offset, "The stack rollup time zone offset must be between -14:00 and +14:00."); + + string sign = offset < TimeSpan.Zero ? "-" : "+"; + var absolute = offset.Duration(); + return $"{sign}{(int)absolute.TotalHours:00}:{absolute.Minutes:00}"; + } + + private static void AddParameter(ICollection>> parameters, string name, FieldValue value) + => parameters.Add(new KeyValuePair>(name, [value])); + + private static void AddParameter(ICollection>> parameters, string name, ICollection values) + => parameters.Add(new KeyValuePair>(name, values)); + + private sealed record StackRollupSort(string Value, string Metric, bool IsDate, bool Ascending); + private sealed record EventLookupSort(string Value, bool Ascending); + private sealed record StackRollupReadiness(bool IsReady, string Reason); + private sealed record StackRollupCursor(int Version, string Sort, long Metric, string StackId, long UtcStart, long UtcEnd, string Fingerprint); + private sealed record EventLookupCursor(int Version, string Sort, long Date, string EventId, long UtcStart, long UtcEnd, string Fingerprint); + private sealed record EventLookupEsqlRow(string EventId, DateTime Date, long? TotalEvents); + private sealed record StackRollupEsqlRow( + string StackId, + long Total, + long Users, + DateTime FirstOccurrence, + DateTime LastOccurrence, + long? TotalStacks); +} diff --git a/src/Exceptionless.Web/Api/Endpoints/EventEndpoints.cs b/src/Exceptionless.Web/Api/Endpoints/EventEndpoints.cs index 05ad674605..ca1e687064 100644 --- a/src/Exceptionless.Web/Api/Endpoints/EventEndpoints.cs +++ b/src/Exceptionless.Web/Api/Endpoints/EventEndpoints.cs @@ -39,7 +39,7 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["aggregations"] = "A list of values you want returned. Example: avg:value cardinality:value sum:users max:value min:value", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole event object will be returned. If the mode is set to summary than a lightweight object will be returned.", + ["mode"] = "Set to stack to calculate the stack-list dashboard metrics with a lookup join.", }, ResponseDescriptions = new() { ["400"] = "Invalid filter.", @@ -61,7 +61,7 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["aggregations"] = "A list of values you want returned. Example: avg:value cardinality:value sum:users max:value min:value", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole event object will be returned. If the mode is set to summary than a lightweight object will be returned.", + ["mode"] = "Set to stack to calculate the stack-list dashboard metrics with a lookup join.", }, ResponseDescriptions = new() { ["400"] = "Invalid filter.", @@ -83,7 +83,7 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["aggregations"] = "A list of values you want returned. Example: avg:value cardinality:value sum:users max:value min:value", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If mode is set to stack_new, then additional filters will be added.", + ["mode"] = "Set to stack to calculate the stack-list dashboard metrics with a lookup join.", }, ResponseDescriptions = new() { ["400"] = "Invalid filter.", @@ -129,7 +129,7 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["sort"] = "Controls the sort order that the data is returned in. In this example -date returns the results descending by date.", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole event object will be returned. If the mode is set to summary than a lightweight object will be returned.", + ["mode"] = "If no mode is set, the whole event object is returned. Summary returns a lightweight event object; stack returns event metrics grouped into stack summaries.", ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", ["before"] = "The before parameter is a cursor used for pagination and defines your place in the list of results.", @@ -158,7 +158,7 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["sort"] = "Controls the sort order that the data is returned in. In this example -date returns the results descending by date.", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole event object will be returned. If the mode is set to summary than a lightweight object will be returned.", + ["mode"] = "If no mode is set, the whole event object is returned. Summary returns a lightweight event object; stack returns event metrics grouped into stack summaries.", ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", ["before"] = "The before parameter is a cursor used for pagination and defines your place in the list of results.", @@ -188,7 +188,7 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["sort"] = "Controls the sort order that the data is returned in. In this example -date returns the results descending by date.", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole event object will be returned. If the mode is set to summary than a lightweight object will be returned.", + ["mode"] = "If no mode is set, the whole event object is returned. Summary returns a lightweight event object; stack returns event metrics grouped into stack summaries.", ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", ["before"] = "The before parameter is a cursor used for pagination and defines your place in the list of results.", @@ -218,7 +218,7 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["sort"] = "Controls the sort order that the data is returned in. In this example -date returns the results descending by date.", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole event object will be returned. If the mode is set to summary than a lightweight object will be returned.", + ["mode"] = "If no mode is set, the whole event object is returned. Summary returns a lightweight event object; stack returns event metrics grouped into stack summaries.", ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", ["before"] = "The before parameter is a cursor used for pagination and defines your place in the list of results.", @@ -244,7 +244,7 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ParameterDescriptions = new() { ["referenceId"] = "An identifier used that references an event instance.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole event object will be returned. If the mode is set to summary than a lightweight object will be returned.", + ["mode"] = "If no mode is set, the whole event object is returned. Summary returns a lightweight event object; stack returns event metrics grouped into stack summaries.", ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", ["before"] = "The before parameter is a cursor used for pagination and defines your place in the list of results.", @@ -271,7 +271,7 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["referenceId"] = "An identifier used that references an event instance.", ["projectId"] = "The identifier of the project.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole event object will be returned. If the mode is set to summary than a lightweight object will be returned.", + ["mode"] = "If no mode is set, the whole event object is returned. Summary returns a lightweight event object; stack returns event metrics grouped into stack summaries.", ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", ["before"] = "The before parameter is a cursor used for pagination and defines your place in the list of results.", @@ -300,7 +300,7 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["sort"] = "Controls the sort order that the data is returned in. In this example -date returns the results descending by date.", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole event object will be returned. If the mode is set to summary than a lightweight object will be returned.", + ["mode"] = "If no mode is set, the whole event object is returned. Summary returns a lightweight event object; stack returns event metrics grouped into stack summaries.", ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", ["before"] = "The before parameter is a cursor used for pagination and defines your place in the list of results.", @@ -330,7 +330,7 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["sort"] = "Controls the sort order that the data is returned in. In this example -date returns the results descending by date.", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole event object will be returned. If the mode is set to summary than a lightweight object will be returned.", + ["mode"] = "If no mode is set, the whole event object is returned. Summary returns a lightweight event object; stack returns event metrics grouped into stack summaries.", ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", ["before"] = "The before parameter is a cursor used for pagination and defines your place in the list of results.", @@ -358,7 +358,7 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["sort"] = "Controls the sort order that the data is returned in. In this example -date returns the results descending by date.", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole event object will be returned. If the mode is set to summary than a lightweight object will be returned.", + ["mode"] = "If no mode is set, the whole event object is returned. Summary returns a lightweight event object; stack returns event metrics grouped into stack summaries.", ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", ["before"] = "The before parameter is a cursor used for pagination and defines your place in the list of results.", @@ -387,7 +387,7 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["sort"] = "Controls the sort order that the data is returned in. In this example -date returns the results descending by date.", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole event object will be returned. If the mode is set to summary than a lightweight object will be returned.", + ["mode"] = "If no mode is set, the whole event object is returned. Summary returns a lightweight event object; stack returns event metrics grouped into stack summaries.", ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", ["before"] = "The before parameter is a cursor used for pagination and defines your place in the list of results.", @@ -417,7 +417,7 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["sort"] = "Controls the sort order that the data is returned in. In this example -date returns the results descending by date.", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole event object will be returned. If the mode is set to summary than a lightweight object will be returned.", + ["mode"] = "If no mode is set, the whole event object is returned. Summary returns a lightweight event object; stack returns event metrics grouped into stack summaries.", ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", ["before"] = "The before parameter is a cursor used for pagination and defines your place in the list of results.", diff --git a/src/Exceptionless.Web/Api/Endpoints/StackEndpoints.cs b/src/Exceptionless.Web/Api/Endpoints/StackEndpoints.cs index c88d1e757a..6f7449415e 100644 --- a/src/Exceptionless.Web/Api/Endpoints/StackEndpoints.cs +++ b/src/Exceptionless.Web/Api/Endpoints/StackEndpoints.cs @@ -2,6 +2,7 @@ using Exceptionless.Core.Authorization; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; +using Exceptionless.Core.Services; using Exceptionless.Web.Api.Filters; using Exceptionless.Web.Api.Infrastructure; using Exceptionless.Web.Api.Messages; @@ -248,8 +249,8 @@ public static IEndpointRouteBuilder MapStackEndpoints(this IEndpointRouteBuilder }); // Get all - group.MapGet("stacks", async (HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? sort = null, string? time = null, string? offset = null, string? mode = null, int page = 1, int limit = 10) - => (await mediator.InvokeAsync>>(new GetAllStacks(filter, sort, time, offset, mode, page, limit, httpContext))).ToHttpResult(resultMapper)) + group.MapGet("stacks", async (HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? sort = null, string? time = null, string? offset = null, int limit = 10, string? before = null, string? after = null) + => (await mediator.InvokeAsync>>(new GetAllStacks(filter, sort, time, offset, limit, before, after, httpContext))).ToHttpResult(resultMapper)) .RequireAuthorization(AuthorizationRoles.StacksReadPolicy) .Produces>() .ProducesProblem(StatusCodes.Status400BadRequest) @@ -261,9 +262,9 @@ public static IEndpointRouteBuilder MapStackEndpoints(this IEndpointRouteBuilder ["sort"] = "Controls the sort order that the data is returned in. In this example -date returns the results descending by date.", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole stack object will be returned. If the mode is set to summary than a lightweight object will be returned.", - ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", + ["before"] = "A cursor that returns the previous page for this exact filter and sort.", + ["after"] = "A cursor that returns the next page for this exact filter and sort.", }, ResponseDescriptions = new() { ["400"] = "Invalid filter.", @@ -272,8 +273,8 @@ public static IEndpointRouteBuilder MapStackEndpoints(this IEndpointRouteBuilder }); // Get by organization - group.MapGet("organizations/{organizationId:objectid}/stacks", async (string organizationId, HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? sort = null, string? time = null, string? offset = null, string? mode = null, int page = 1, int limit = 10) - => (await mediator.InvokeAsync>>(new GetStacksByOrganization(organizationId, filter, sort, time, offset, mode, page, limit, httpContext))).ToHttpResult(resultMapper)) + group.MapGet("organizations/{organizationId:objectid}/stacks", async (string organizationId, HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? sort = null, string? time = null, string? offset = null, int limit = 10, string? before = null, string? after = null) + => (await mediator.InvokeAsync>>(new GetStacksByOrganization(organizationId, filter, sort, time, offset, limit, before, after, httpContext))).ToHttpResult(resultMapper)) .RequireAuthorization(AuthorizationRoles.StacksReadPolicy) .Produces>() .ProducesProblem(StatusCodes.Status400BadRequest) @@ -287,9 +288,9 @@ public static IEndpointRouteBuilder MapStackEndpoints(this IEndpointRouteBuilder ["sort"] = "Controls the sort order that the data is returned in. In this example -date returns the results descending by date.", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole stack object will be returned. If the mode is set to summary than a lightweight object will be returned.", - ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", + ["before"] = "A cursor that returns the previous page for this exact filter and sort.", + ["after"] = "A cursor that returns the next page for this exact filter and sort.", }, ResponseDescriptions = new() { ["400"] = "Invalid filter.", @@ -299,8 +300,8 @@ public static IEndpointRouteBuilder MapStackEndpoints(this IEndpointRouteBuilder }); // Get by project - group.MapGet("projects/{projectId:objectid}/stacks", async (string projectId, HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? sort = null, string? time = null, string? offset = null, string? mode = null, int page = 1, int limit = 10) - => (await mediator.InvokeAsync>>(new GetStacksByProject(projectId, filter, sort, time, offset, mode, page, limit, httpContext))).ToHttpResult(resultMapper)) + group.MapGet("projects/{projectId:objectid}/stacks", async (string projectId, HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? sort = null, string? time = null, string? offset = null, int limit = 10, string? before = null, string? after = null) + => (await mediator.InvokeAsync>>(new GetStacksByProject(projectId, filter, sort, time, offset, limit, before, after, httpContext))).ToHttpResult(resultMapper)) .RequireAuthorization(AuthorizationRoles.StacksReadPolicy) .Produces>() .ProducesProblem(StatusCodes.Status400BadRequest) @@ -314,9 +315,9 @@ public static IEndpointRouteBuilder MapStackEndpoints(this IEndpointRouteBuilder ["sort"] = "Controls the sort order that the data is returned in. In this example -date returns the results descending by date.", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole stack object will be returned. If the mode is set to summary than a lightweight object will be returned.", - ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", + ["before"] = "A cursor that returns the previous page for this exact filter and sort.", + ["after"] = "A cursor that returns the next page for this exact filter and sort.", }, ResponseDescriptions = new() { ["400"] = "Invalid filter.", diff --git a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs index c1bdb6d5c1..cd51cb2882 100644 --- a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs @@ -38,6 +38,7 @@ public class EventHandler( IOrganizationRepository organizationRepository, IProjectRepository projectRepository, IStackRepository stackRepository, + IStackRollupSearchService stackRollupSearchService, EventPostService eventPostService, IQueue eventUserDescriptionQueue, MiniValidationValidator miniValidationValidator, @@ -45,7 +46,7 @@ public class EventHandler( ICacheClient cacheClient, ITextSerializer serializer, PersistentEventQueryValidator validator, - EventStackQueryValidator stackModeValidator, + EventStackQueryValidator stackValidator, AppOptions appOptions, UsageService usageService, TimeProvider timeProvider, @@ -166,7 +167,7 @@ public async Task>> Handle(GetAllEvents message) var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organizations.GetRetentionUtcCutoff(appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(organizations) { IsUserOrganizationsFilter = true }; - return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include), timeExpression: message.Time); } public async Task>> Handle(GetEventsByOrganization message) @@ -181,7 +182,7 @@ public async Task>> Handle(GetEventsByOrganization me var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(organization); - return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include), timeExpression: message.Time); } public async Task>> Handle(GetEventsByProject message) @@ -200,7 +201,7 @@ public async Task>> Handle(GetEventsByProject message var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(project, appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(project, organization); - return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include), timeExpression: message.Time); } public async Task>> Handle(GetEventsByStack message) @@ -219,7 +220,7 @@ public async Task>> Handle(GetEventsByStack message) var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(stack, appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(stack, organization); - return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include), timeExpression: message.Time); } public async Task>> Handle(GetEventsByReferenceId message) @@ -262,7 +263,7 @@ public async Task>> Handle(GetEventsBySessionId messa var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organizations.GetRetentionUtcCutoff(appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(organizations) { IsUserOrganizationsFilter = true }; - return await GetInternalAsync(sf, ti, httpContext, $"(reference:{message.SessionId} OR ref.session:{message.SessionId}) {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, $"(reference:{message.SessionId} OR ref.session:{message.SessionId}) {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include), timeExpression: message.Time); } public async Task>> Handle(GetEventsBySessionIdAndProject message) @@ -281,7 +282,7 @@ public async Task>> Handle(GetEventsBySessionIdAndPro var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(project, appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(project, organization); - return await GetInternalAsync(sf, ti, httpContext, $"(reference:{message.SessionId} OR ref.session:{message.SessionId}) {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, $"(reference:{message.SessionId} OR ref.session:{message.SessionId}) {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include), timeExpression: message.Time); } public async Task>> Handle(GetSessions message) @@ -293,7 +294,7 @@ public async Task>> Handle(GetSessions message) var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organizations.GetRetentionUtcCutoff(appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(organizations) { IsUserOrganizationsFilter = true }; - return await GetInternalAsync(sf, ti, httpContext, $"type:{Event.KnownTypes.Session} {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, $"type:{Event.KnownTypes.Session} {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include), timeExpression: message.Time); } public async Task>> Handle(GetSessionsByOrganization message) @@ -308,7 +309,7 @@ public async Task>> Handle(GetSessionsByOrganization var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(organization); - return await GetInternalAsync(sf, ti, httpContext, $"type:{Event.KnownTypes.Session} {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, $"type:{Event.KnownTypes.Session} {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include), timeExpression: message.Time); } public async Task>> Handle(GetSessionsByProject message) @@ -327,7 +328,7 @@ public async Task>> Handle(GetSessionsByProject messa var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(project, appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(project, organization); - return await GetInternalAsync(sf, ti, httpContext, $"type:{Event.KnownTypes.Session} {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, $"type:{Event.KnownTypes.Session} {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include), timeExpression: message.Time); } public async Task Handle(SetEventUserDescription message) @@ -690,7 +691,13 @@ public async Task> Handle(DeleteEvents message) private async Task> CountInternalAsync(AppFilter sf, TimeInfo ti, HttpContext httpContext, string? filter = null, string? aggregations = null, string? mode = null) { - var pr = await GetQueryValidator(mode).ValidateQueryAsync(filter); + bool isStackMode = String.Equals(mode, "stack", StringComparison.OrdinalIgnoreCase); + if (mode is not null && !isStackMode) + return Result.BadRequest("Mode must be 'stack' when specified."); + + var pr = isStackMode + ? await stackValidator.ValidateQueryAsync(filter) + : await validator.ValidateQueryAsync(filter); if (!pr.IsValid) return Result.BadRequest(pr.Message ?? "Invalid filter."); @@ -698,14 +705,11 @@ private async Task> CountInternalAsync(AppFilter sf, TimeInf if (!far.IsValid) return Result.BadRequest(far.Message ?? "Invalid aggregations."); - sf.UsesPremiumFeatures = pr.UsesPremiumFeatures || far.UsesPremiumFeatures; + sf.UsesPremiumFeatures = pr.UsesPremiumFeatures || !isStackMode && far.UsesPremiumFeatures; AppFilter? systemFilter = ApiFilterPolicy.ShouldApplySystemFilter(sf, filter, httpContext.Request) ? sf : null; if (systemFilter is not null && ApiFilterPolicy.IsPremiumFeatureQueryBlocked(systemFilter)) return PlanLimitResult(ApiFilterPolicy.PremiumSearchUpgradeMessage); - if (mode == "stack_new") - filter = AddFirstOccurrenceFilter(ti.Range, filter); - var query = new RepositoryQuery() .AppFilter(systemFilter) .DateRange(ti.Range.UtcStart, ti.Range.UtcEnd, ti.Field) @@ -714,7 +718,24 @@ private async Task> CountInternalAsync(AppFilter sf, TimeInf CountResult result; try { - result = await eventRepository.CountAsync(q => q.SystemFilter(query).FilterExpression(filter).EnforceEventStackFilter().AggregationsExpression(aggregations)); + if (isStackMode || await stackRollupSearchService.RequiresLookupJoinAsync(filter, httpContext.RequestAborted)) + { + result = await stackRollupSearchService.CountEventsAsync(new EventLookupCountRequest( + systemFilter, + ti.Range.UtcStart, + ti.Range.UtcEnd, + ti.Offset, + filter, + aggregations), httpContext.RequestAborted); + } + else + { + result = await eventRepository.CountAsync(q => q.SystemFilter(query).FilterExpression(filter).EnforceEventStackFilter().AggregationsExpression(aggregations)); + } + } + catch (NotSupportedException ex) + { + return Result.BadRequest(ex.Message); } catch (Exception ex) { @@ -728,8 +749,21 @@ private async Task> CountInternalAsync(AppFilter sf, TimeInf return result; } - private async Task>> GetInternalAsync(AppFilter sf, TimeInfo ti, HttpContext httpContext, string? filter = null, string? sort = null, string? mode = null, int? page = null, int limit = 10, string? before = null, string? after = null, string? premiumFeatureUpgradeMessage = null, bool includeTotal = false) + private async Task>> GetInternalAsync(AppFilter sf, TimeInfo ti, HttpContext httpContext, string? filter = null, string? sort = null, string? mode = null, int? page = null, int limit = 10, string? before = null, string? after = null, string? premiumFeatureUpgradeMessage = null, bool includeTotal = false, string? timeExpression = null) { + if (mode is not null + && !String.Equals(mode, "summary", StringComparison.OrdinalIgnoreCase) + && !String.Equals(mode, "stack", StringComparison.OrdinalIgnoreCase)) + return Result.BadRequest("Mode must be 'summary' or 'stack' when specified."); + + if (String.Equals(mode, "stack", StringComparison.OrdinalIgnoreCase)) + { + if (page.HasValue) + return Result.BadRequest("Stack mode uses before and after cursors; page is not supported."); + + return await GetStackModeEventsInternalAsync(sf, ti, httpContext, filter, sort, limit, before, after, includeTotal, timeExpression); + } + var currentUser = httpContext.Request.GetUser(); using var _ = _logger.BeginScope(new ExceptionlessState() .Property("Search Filter", new @@ -755,7 +789,7 @@ private async Task>> GetInternalAsync(AppFilter sf, T if (skip > Pagination.MaximumSkip) return new PagedResult(Array.Empty(), false); - var pr = await GetQueryValidator(mode).ValidateQueryAsync(filter); + var pr = await validator.ValidateQueryAsync(filter); if (!pr.IsValid) return Result.BadRequest(pr.Message ?? "Invalid filter."); @@ -766,6 +800,48 @@ private async Task>> GetInternalAsync(AppFilter sf, T try { + if (await stackRollupSearchService.RequiresLookupJoinAsync(filter, httpContext.RequestAborted)) + { + if (page.HasValue) + return Result.BadRequest("Event queries with stack filters use before and after cursors; page is not supported."); + if (before is not null && after is not null) + return Result.BadRequest("The before and after parameters cannot be used together."); + + EventLookupSearchResult lookup; + try + { + lookup = await stackRollupSearchService.SearchEventsAsync(new EventLookupSearchRequest( + appliedAppFilter, + ti.Range.UtcStart, + ti.Range.UtcEnd, + timeExpression, + filter, + sort, + limit, + before, + after, + includeTotal), httpContext.RequestAborted); + } + catch (InvalidEventLookupCursorException ex) + { + return Result.BadRequest(ex.Message); + } + catch (ArgumentOutOfRangeException ex) when (ex.ParamName == "sort") + { + return Result.BadRequest("Event queries with stack filters can only be sorted by date or -date."); + } + + var byId = (await eventRepository.GetByIdsAsync(lookup.EventIds.ToArray())).ToDictionary(eventItem => eventItem.Id, StringComparer.Ordinal); + var documents = lookup.EventIds.Where(byId.ContainsKey).Select(id => byId[id]).ToList(); + if (String.Equals(mode, "summary", StringComparison.OrdinalIgnoreCase)) + { + var summaries = await GetEventSummariesAsync(documents); + return new PagedResult(summaries.Cast().ToList(), lookup.HasMore, null, lookup.Total, lookup.Before, lookup.After); + } + + return new PagedResult(documents.Cast().ToList(), lookup.HasMore, null, lookup.Total, lookup.Before, lookup.After); + } + FindResults events; switch (mode) { @@ -790,54 +866,6 @@ private async Task>> GetInternalAsync(AppFilter sf, T }; }).ToList(); return new PagedResult(summaries.Cast().ToList(), events.HasMore && !Pagination.NextPageExceedsSkipLimit(page, limit), page, includeTotal ? events.Total : null, events.Hits.FirstOrDefault()?.GetSortToken(serializer), events.Hits.LastOrDefault()?.GetSortToken(serializer)); - case "stack_recent": - case "stack_frequent": - case "stack_new": - case "stack_users": - if (!String.IsNullOrEmpty(sort)) - return Result.BadRequest("Sort is not supported in stack mode."); - - var systemFilter = new RepositoryQuery() - .AppFilter(appliedAppFilter) - .EnforceEventStackFilter() - .DateRange(ti.Range.UtcStart, ti.Range.UtcEnd, (PersistentEvent e) => e.Date) - .Index(ti.Range.UtcStart, ti.Range.UtcEnd); - - string? stackAggregations = mode switch - { - "stack_recent" => "cardinality:user sum:count~1 min:date -max:date", - "stack_frequent" => "cardinality:user -sum:count~1 min:date max:date", - "stack_new" => "cardinality:user sum:count~1 -min:date max:date", - "stack_users" => "-cardinality:user sum:count~1 min:date max:date", - _ => null - }; - - if (mode == "stack_new") - filter = AddFirstOccurrenceFilter(ti.Range, filter); - - string aggregationExpression = includeTotal - ? $"cardinality:stack_id terms:(stack_id~{Pagination.GetSkip(resolvedPage + 1, limit) + 1} {stackAggregations})" - : $"terms:(stack_id~{Pagination.GetSkip(resolvedPage + 1, limit) + 1} {stackAggregations})"; - - var countResponse = await eventRepository.CountAsync(q => q - .SystemFilter(systemFilter) - .FilterExpression(filter) - .EnforceEventStackFilter() - .AggregationsExpression(aggregationExpression), - o => o.TrackTotalHits(false)); - - var stackTerms = countResponse.Aggregations.Terms("terms_stack_id"); - if (stackTerms is null || stackTerms.Buckets.Count == 0) - return new PagedResult(Array.Empty(), false); - - string[] stackIds = stackTerms.Buckets.Skip(skip).Take(limit + 1).Select(t => t.Key).ToArray(); - var stacks = (await stackRepository.GetByIdsAsync(stackIds)).Select(s => s.ApplyOffset(ti.Offset)).ToList(); - - var stackSummaries = await GetStackSummariesAsync(stacks, stackTerms.Buckets, sf, ti); - - double? totalStackCount = countResponse.Aggregations.Cardinality("cardinality_stack_id")?.Value; - long? total = includeTotal && totalStackCount.HasValue ? Convert.ToInt64(totalStackCount.Value) : null; - return new PagedResult(stackSummaries.Take(limit).Cast().ToList(), stackSummaries.Count > limit && !Pagination.NextPageExceedsSkipLimit(resolvedPage, limit), resolvedPage, total); default: events = await GetEventsInternalAsync(appliedAppFilter, ti, filter, sort, page, limit, before, after, includeTotal); return new PagedResult(events.Documents.Cast().ToList(), events.HasMore && !Pagination.NextPageExceedsSkipLimit(page, limit), page, includeTotal ? events.Total : null, events.Hits.FirstOrDefault()?.GetSortToken(serializer), events.Hits.LastOrDefault()?.GetSortToken(serializer)); @@ -854,50 +882,6 @@ private async Task>> GetInternalAsync(AppFilter sf, T } } - private static string AddFirstOccurrenceFilter(DateTimeRange timeRange, string? filter) - { - bool inverted = false; - if (filter is not null && filter.StartsWith("@!")) - { - inverted = true; - filter = filter.Substring(2); - } - - var sb = new StringBuilder(); - if (inverted) - sb.Append("@!"); - - sb.Append("first_occurrence:[\""); - sb.Append(timeRange.UtcStart.ToString("O")); - sb.Append("\" TO \""); - sb.Append(timeRange.UtcEnd.ToString("O")); - sb.Append("\"]"); - - if (String.IsNullOrEmpty(filter)) - return sb.ToString(); - - sb.Append(' '); - - bool isGrouped = filter.StartsWith('(') && filter.EndsWith(')'); - - if (isGrouped) - sb.Append(filter); - else - sb.Append('(').Append(filter).Append(')'); - - return sb.ToString(); - } - - private static bool IsStackMode(string? mode) - { - return mode is "stack_recent" or "stack_frequent" or "stack_new" or "stack_users"; - } - - private IAppQueryValidator GetQueryValidator(string? mode) - { - return IsStackMode(mode) ? stackModeValidator : validator; - } - private Task> GetEventsInternalAsync(AppFilter? systemFilter, TimeInfo ti, string? filter, string? sort, int? page, int limit, string? before, string? after, bool includeTotal) { if (String.IsNullOrEmpty(sort)) @@ -915,18 +899,102 @@ private Task> GetEventsInternalAsync(AppFilter? sys : o.SearchBeforeToken(before, serializer).SearchAfterToken(after, serializer).PageLimit(limit).TrackTotalHits(includeTotal)); } - private async Task> GetStackSummariesAsync(List stacks, IReadOnlyCollection> stackTerms, AppFilter sf, TimeInfo ti) + private async Task> GetEventSummariesAsync(IReadOnlyCollection events) + { + var projects = await projectRepository.GetByIdsAsync(events.Select(eventItem => eventItem.ProjectId).Distinct().ToArray(), query => query.Cache()); + var projectNames = projects.ToDictionary(project => project.Id, project => project.Name); + return events.Select(eventItem => + { + var summaryData = formattingPluginManager.GetEventSummaryData(eventItem); + return new EventSummaryModel + { + Id = summaryData.Id, + TemplateKey = summaryData.TemplateKey, + Date = eventItem.Date, + ProjectId = eventItem.ProjectId, + ProjectName = projectNames.GetValueOrDefault(eventItem.ProjectId), + Tags = eventItem.Tags?.OfType().Order(StringComparer.OrdinalIgnoreCase).ToArray() ?? [], + Type = eventItem.Type, + Version = eventItem.GetVersion(), + Data = summaryData.Data + }; + }).ToList(); + } + + private async Task>> GetStackModeEventsInternalAsync( + AppFilter appFilter, + TimeInfo time, + HttpContext httpContext, + string? filter, + string? sort, + int limit, + string? before, + string? after, + bool includeTotal, + string? timeExpression) + { + if (before is not null && after is not null) + return Result.BadRequest("The before and after parameters cannot be used together."); + + limit = Pagination.GetLimit(limit); + var validation = await stackValidator.ValidateQueryAsync(filter); + if (!validation.IsValid) + return Result.BadRequest(validation.Message ?? "Invalid filter."); + + appFilter.UsesPremiumFeatures = validation.UsesPremiumFeatures; + AppFilter? appliedAppFilter = ApiFilterPolicy.ShouldApplySystemFilter(appFilter, filter, httpContext.Request) ? appFilter : null; + if (appliedAppFilter is not null && ApiFilterPolicy.IsPremiumFeatureQueryBlocked(appliedAppFilter)) + return PlanLimitResult>(ApiFilterPolicy.PremiumSearchUpgradeMessage); + + try + { + var result = await stackRollupSearchService.SearchAsync(new StackRollupSearchRequest( + appliedAppFilter, + time.Range.UtcStart, + time.Range.UtcEnd, + time.Offset, + timeExpression, + filter, + sort, + limit, + before, + after, + includeTotal), httpContext.RequestAborted); + + string[] stackIds = result.Rows.Select(row => row.StackId).ToArray(); + var stacks = (await stackRepository.GetByIdsAsync(stackIds)) + .Select(stack => stack.ApplyOffset(time.Offset)) + .ToList(); + var summaries = await GetStackSummariesAsync(stacks, result.Rows, appFilter, time); + return new PagedResult(summaries.Cast().ToList(), result.HasMore, null, result.Total, result.Before, result.After); + } + catch (InvalidStackRollupCursorException ex) + { + return Result.BadRequest(ex.Message); + } + catch (ArgumentOutOfRangeException ex) when (ex.ParamName == "sort") + { + return Result.BadRequest("Stack mode sort must be one of total, users, first_occurrence, or last_occurrence, optionally prefixed with '-'."); + } + } + + private async Task> GetStackSummariesAsync(List stacks, IReadOnlyCollection rows, AppFilter appFilter, TimeInfo time) { if (stacks.Count == 0) - return new List(0); + return []; - var projects = await projectRepository.GetByIdsAsync(stacks.Select(s => s.ProjectId).Distinct().ToArray(), o => o.Cache()); - var projectNames = projects.ToDictionary(p => p.Id, p => p.Name); - var totalUsers = await GetUserCountByProjectIdsAsync(stacks, sf, ti.Range.UtcStart, ti.Range.UtcEnd); - return stacks.Join(stackTerms, s => s.Id, tk => tk.Key, (stack, term) => + var stacksById = stacks.ToDictionary(stack => stack.Id, StringComparer.Ordinal); + var projects = await projectRepository.GetByIdsAsync(stacks.Select(stack => stack.ProjectId).Distinct().ToArray(), query => query.Cache()); + var projectNames = projects.ToDictionary(project => project.Id, project => project.Name); + var totalUsers = await GetUserCountByProjectIdsAsync(stacks, appFilter, time.Range.UtcStart, time.Range.UtcEnd); + var summaries = new List(rows.Count); + foreach (var row in rows) { + if (!stacksById.TryGetValue(row.StackId, out var stack)) + continue; + var data = formattingPluginManager.GetStackSummaryData(stack); - var summary = new StackSummaryModel + summaries.Add(new StackSummaryModel { Id = data.Id, TemplateKey = data.TemplateKey, @@ -936,40 +1004,38 @@ private async Task> GetStackSummariesAsync(List().Order(StringComparer.OrdinalIgnoreCase).ToArray() ?? [], Title = stack.Title, Status = stack.Status, - FirstOccurrence = term.Aggregations.Min("min_date")?.Value ?? stack.FirstOccurrence, - LastOccurrence = term.Aggregations.Max("max_date")?.Value ?? stack.LastOccurrence, - Total = (long)(term.Aggregations.Sum("sum_count")?.Value ?? term.Total.GetValueOrDefault()), - - Users = term.Aggregations.Cardinality("cardinality_user")?.Value.GetValueOrDefault() ?? 0, + FirstOccurrence = row.FirstOccurrence, + LastOccurrence = row.LastOccurrence, + Total = row.Total, + Users = row.Users, TotalUsers = totalUsers.GetOrDefault(stack.ProjectId) - }; + }); + } - return summary; - }).ToList(); + return summaries; } - private async Task> GetUserCountByProjectIdsAsync(ICollection stacks, AppFilter sf, DateTime utcStart, DateTime utcEnd) + private async Task> GetUserCountByProjectIdsAsync(ICollection stacks, AppFilter appFilter, DateTime utcStart, DateTime utcEnd) { using var scopedCacheClient = new ScopedCacheClient(cacheClient, $"Project:user-count:{utcStart.Floor(TimeSpan.FromMinutes(15)).Ticks}-{utcEnd.Floor(TimeSpan.FromMinutes(15)).Ticks}"); - var projectIds = stacks.Select(s => s.ProjectId).Distinct().ToList(); + var projectIds = stacks.Select(stack => stack.ProjectId).Distinct().ToList(); var cachedTotals = await scopedCacheClient.GetAllAsync(projectIds); - - var totals = cachedTotals.Where(kvp => kvp.Value.HasValue).ToDictionary(kvp => kvp.Key, kvp => kvp.Value.Value); + var totals = cachedTotals.Where(item => item.Value.HasValue).ToDictionary(item => item.Key, item => item.Value.Value); if (totals.Count == projectIds.Count) return totals; - var systemFilter = new RepositoryQuery().AppFilter(sf).DateRange(utcStart, utcEnd, (PersistentEvent e) => e.Date).Index(utcStart, utcEnd); var projects = cachedTotals - .Where(kvp => !kvp.Value.HasValue && stacks.Contains(s => s.ProjectId == kvp.Key)) - .Select(kvp => new Project { Id = kvp.Key, OrganizationId = stacks.First(s => s.ProjectId == kvp.Key).OrganizationId }) + .Where(item => !item.Value.HasValue && stacks.Contains(stack => stack.ProjectId == item.Key)) + .Select(item => new Project { Id = item.Key, OrganizationId = stacks.First(stack => stack.ProjectId == item.Key).OrganizationId }) .ToList(); - var countResult = await eventRepository.CountAsync(q => q.SystemFilter(systemFilter).FilterExpression(projects.BuildFilter()).EnforceEventStackFilter().AggregationsExpression("terms:(project_id cardinality:user)")); - - var projectTerms = countResult.Aggregations.Terms("terms_project_id")?.Buckets ?? []; - var aggregations = projectTerms.ToDictionary(t => t.Key, t => t.Aggregations.Cardinality("cardinality_user")?.Value.GetValueOrDefault() ?? 0); - await scopedCacheClient.SetAllAsync(aggregations.Where(t => t.Value >= 10).ToDictionary(k => k.Key, v => v.Value), TimeSpan.FromMinutes(5)); + var aggregations = (await stackRollupSearchService.GetProjectUserCountsAsync(new StackRollupProjectUsersRequest( + appFilter, + utcStart, + utcEnd, + projects.Select(project => project.Id).ToArray()))) + .ToDictionary(item => item.Key, item => (double)item.Value); + await scopedCacheClient.SetAllAsync(aggregations.Where(item => item.Value >= 10).ToDictionary(item => item.Key, item => item.Value), TimeSpan.FromMinutes(5)); totals.AddRange(aggregations); - return totals; } diff --git a/src/Exceptionless.Web/Api/Handlers/StackHandler.cs b/src/Exceptionless.Web/Api/Handlers/StackHandler.cs index 0ac1975ef6..0fb87dff40 100644 --- a/src/Exceptionless.Web/Api/Handlers/StackHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/StackHandler.cs @@ -22,8 +22,10 @@ using Foundatio.Mediator; using Foundatio.Queues; using Foundatio.Repositories; +using Foundatio.Repositories.Elasticsearch.Extensions; using Foundatio.Repositories.Extensions; using Foundatio.Repositories.Models; +using Foundatio.Serializer; using McSherry.SemanticVersioning; namespace Exceptionless.Web.Api.Handlers; @@ -32,14 +34,12 @@ public class StackHandler( IStackRepository stackRepository, IOrganizationRepository organizationRepository, IProjectRepository projectRepository, - IEventRepository eventRepository, IWebHookRepository webHookRepository, WebHookDataPluginManager webHookDataPluginManager, IQueue webHookNotificationQueue, - ICacheClient cacheClient, - FormattingPluginManager formattingPluginManager, SemanticVersionParser semanticVersionParser, StackQueryValidator validator, + ITextSerializer serializer, AppOptions options, TimeProvider timeProvider, ILoggerFactory loggerFactory) @@ -358,7 +358,7 @@ public async Task>> Handle(GetAllStacks message) var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organizations.GetRetentionUtcCutoff(options.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(organizations) { IsUserOrganizationsFilter = true }; - return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit); + return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Limit, message.Before, message.After); } public async Task>> Handle(GetStacksByOrganization message) @@ -373,7 +373,7 @@ public async Task>> Handle(GetStacksByOrganization me var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(options.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(organization); - return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit); + return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Limit, message.Before, message.After); } public async Task>> Handle(GetStacksByProject message) @@ -392,16 +392,16 @@ public async Task>> Handle(GetStacksByProject message var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(project, options.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(project, organization); - return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit); + return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Limit, message.Before, message.After); } - private async Task>> GetInternalAsync(AppFilter sf, TimeInfo ti, HttpContext httpContext, string? filter = null, string? sort = null, string? mode = null, int page = 1, int limit = 10) + private async Task>> GetInternalAsync(AppFilter sf, TimeInfo ti, HttpContext httpContext, string? filter = null, string? sort = null, int limit = 10, string? before = null, string? after = null) { - page = Pagination.GetPage(page); + if (before is not null && after is not null) + return Result.BadRequest("The before and after parameters cannot be used together."); + limit = Pagination.GetLimit(limit); - int skip = Pagination.GetSkip(page, limit); - if (skip > Pagination.MaximumSkip) - return new PagedResult(Array.Empty(), false); + sort = String.IsNullOrWhiteSpace(sort) ? "-last" : sort; var pr = await validator.ValidateQueryAsync(filter); if (!pr.IsValid) @@ -414,93 +414,29 @@ private async Task>> GetInternalAsync(AppFilter sf, T try { - var results = await stackRepository.FindAsync(q => q.AppFilter(systemFilter).FilterExpression(filter).SortExpression(sort).DateRange(ti.Range.UtcStart, ti.Range.UtcEnd, ti.Field), o => o.PageNumber(page).PageLimit(limit)); + var results = await stackRepository.FindAsync( + q => q.AppFilter(systemFilter).FilterExpression(filter).SortExpression(sort).DateRange(ti.Range.UtcStart, ti.Range.UtcEnd, ti.Field), + o => o.SearchBeforeToken(before, serializer).SearchAfterToken(after, serializer).PageLimit(limit)); var stacks = results.Documents.Select(s => s.ApplyOffset(ti.Offset)).ToList(); - if (!String.IsNullOrEmpty(mode) && String.Equals(mode, "summary", StringComparison.OrdinalIgnoreCase)) - return new PagedResult((await GetStackSummariesAsync(stacks, sf, ti)).Cast().ToList(), results.HasMore && !Pagination.NextPageExceedsSkipLimit(page, limit), page); - - return new PagedResult(stacks.Cast().ToList(), results.HasMore && !Pagination.NextPageExceedsSkipLimit(page, limit), page); + return new PagedResult( + stacks.Cast().ToList(), + results.HasMore, + Page: null, + Total: null, + results.Hits.FirstOrDefault()?.GetSortToken(serializer), + results.Hits.LastOrDefault()?.GetSortToken(serializer)); } catch (ApplicationException ex) { var currentUser = httpContext.Request.GetUser(); - using (_logger.BeginScope(new ExceptionlessState().Property("Search Filter", new { SystemFilter = sf, UserFilter = filter, Time = ti, Page = page, Limit = limit }).Tag("Search").Identity(currentUser?.EmailAddress).Property("User", currentUser).SetHttpContext(httpContext))) + using (_logger.BeginScope(new ExceptionlessState().Property("Search Filter", new { SystemFilter = sf, UserFilter = filter, Time = ti, Sort = sort, Before = before, After = after, Limit = limit }).Tag("Search").Identity(currentUser?.EmailAddress).Property("User", currentUser).SetHttpContext(httpContext))) _logger.LogError(ex, "An error has occurred. Please check your search filter"); throw; } } - private async Task> GetStackSummariesAsync(ICollection stacks, AppFilter eventSystemFilter, TimeInfo ti) - { - if (stacks.Count == 0) - return new List(); - - var systemFilter = new RepositoryQuery().AppFilter(eventSystemFilter).DateRange(ti.Range.UtcStart, ti.Range.UtcEnd, (PersistentEvent e) => e.Date).Index(ti.Range.UtcStart, ti.Range.UtcEnd); - var stackTerms = await eventRepository.CountAsync(q => q.SystemFilter(systemFilter).Stack(stacks.Select(r => r.Id)).AggregationsExpression($"terms:(stack_id~{stacks.Count} cardinality:user sum:count~1 min:date max:date)")); - var buckets = stackTerms.Aggregations.Terms("terms_stack_id")?.Buckets ?? []; - return await GetStackSummariesAsync(stacks, buckets, eventSystemFilter, ti); - } - - private async Task> GetStackSummariesAsync(ICollection stacks, IReadOnlyCollection> stackTerms, AppFilter sf, TimeInfo ti) - { - if (stacks.Count == 0) - return new List(0); - - var projects = await projectRepository.GetByIdsAsync(stacks.Select(s => s.ProjectId).Distinct().ToArray(), o => o.Cache()); - var projectNames = projects.ToDictionary(p => p.Id, p => p.Name); - var totalUsers = await GetUserCountByProjectIdsAsync(stacks, sf, ti.Range.UtcStart, ti.Range.UtcEnd); - return stacks.Join(stackTerms, s => s.Id, tk => tk.Key, (stack, term) => - { - var data = formattingPluginManager.GetStackSummaryData(stack); - var summary = new StackSummaryModel - { - Id = data.Id, - TemplateKey = data.TemplateKey, - Data = data.Data, - ProjectId = stack.ProjectId, - ProjectName = projectNames.GetValueOrDefault(stack.ProjectId), - Tags = stack.Tags?.OfType().Order(StringComparer.OrdinalIgnoreCase).ToArray() ?? [], - Title = stack.Title, - Status = stack.Status, - FirstOccurrence = term.Aggregations.Min("min_date")?.Value ?? stack.FirstOccurrence, - LastOccurrence = term.Aggregations.Max("max_date")?.Value ?? stack.LastOccurrence, - Total = (long)(term.Aggregations.Sum("sum_count")?.Value ?? term.Total.GetValueOrDefault()), - - Users = term.Aggregations.Cardinality("cardinality_user")?.Value.GetValueOrDefault() ?? 0, - TotalUsers = totalUsers.GetOrDefault(stack.ProjectId) - }; - - return summary; - }).ToList(); - } - - private async Task> GetUserCountByProjectIdsAsync(ICollection stacks, AppFilter sf, DateTime utcStart, DateTime utcEnd) - { - using var scopedCacheClient = new ScopedCacheClient(cacheClient, $"Project:user-count:{utcStart.Floor(TimeSpan.FromMinutes(15)).Ticks}-{utcEnd.Floor(TimeSpan.FromMinutes(15)).Ticks}"); - var projectIds = stacks.Select(s => s.ProjectId).Distinct().ToList(); - var cachedTotals = await scopedCacheClient.GetAllAsync(projectIds); - - var totals = cachedTotals.Where(kvp => kvp.Value.HasValue).ToDictionary(kvp => kvp.Key, kvp => kvp.Value.Value); - if (totals.Count == projectIds.Count) - return totals; - - var systemFilter = new RepositoryQuery().AppFilter(sf).DateRange(utcStart, utcEnd, (PersistentEvent e) => e.Date).Index(utcStart, utcEnd); - var projects = cachedTotals - .Where(kvp => !kvp.Value.HasValue && stacks.Contains(s => s.ProjectId == kvp.Key)) - .Select(kvp => new Project { Id = kvp.Key, OrganizationId = stacks.First(s => s.ProjectId == kvp.Key).OrganizationId }) - .ToList(); - var countResult = await eventRepository.CountAsync(q => q.SystemFilter(systemFilter).FilterExpression(projects.BuildFilter()).AggregationsExpression("terms:(project_id cardinality:user)")); - - var projectTerms = countResult.Aggregations.Terms("terms_project_id")?.Buckets ?? []; - var aggregations = projectTerms.ToDictionary(t => t.Key, t => t.Aggregations.Cardinality("cardinality_user")?.Value.GetValueOrDefault() ?? 0); - await scopedCacheClient.SetAllAsync(aggregations.Where(t => t.Value >= 10).ToDictionary(k => k.Key, v => v.Value), TimeSpan.FromMinutes(5)); - totals.AddRange(aggregations); - - return totals; - } - private async Task GetModelAsync(string id, HttpContext httpContext, bool useCache = true) { if (String.IsNullOrEmpty(id)) diff --git a/src/Exceptionless.Web/Api/Messages/StackMessages.cs b/src/Exceptionless.Web/Api/Messages/StackMessages.cs index 26fba9556c..78b02faae0 100644 --- a/src/Exceptionless.Web/Api/Messages/StackMessages.cs +++ b/src/Exceptionless.Web/Api/Messages/StackMessages.cs @@ -16,6 +16,6 @@ public record MarkStacksNotCritical(string Ids, HttpContext Context); public record ChangeStacksStatus(string Ids, StackStatus Status, HttpContext Context); public record PromoteStack(string Id, HttpContext Context); public record DeleteStacks(string Ids, HttpContext Context); -public record GetAllStacks(string? Filter, string? Sort, string? Time, string? Offset, string? Mode, int Page, int Limit, HttpContext Context); -public record GetStacksByOrganization(string OrganizationId, string? Filter, string? Sort, string? Time, string? Offset, string? Mode, int Page, int Limit, HttpContext Context); -public record GetStacksByProject(string ProjectId, string? Filter, string? Sort, string? Time, string? Offset, string? Mode, int Page, int Limit, HttpContext Context); +public record GetAllStacks(string? Filter, string? Sort, string? Time, string? Offset, int Limit, string? Before, string? After, HttpContext Context); +public record GetStacksByOrganization(string OrganizationId, string? Filter, string? Sort, string? Time, string? Offset, int Limit, string? Before, string? After, HttpContext Context); +public record GetStacksByProject(string ProjectId, string? Filter, string? Sort, string? Time, string? Offset, int Limit, string? Before, string? After, HttpContext Context); diff --git a/src/Exceptionless.Web/ClientApp.angular/app/frequent-controller.js b/src/Exceptionless.Web/ClientApp.angular/app/frequent-controller.js index b0e2225202..f73a5799b0 100644 --- a/src/Exceptionless.Web/ClientApp.angular/app/frequent-controller.js +++ b/src/Exceptionless.Web/ClientApp.angular/app/frequent-controller.js @@ -212,10 +212,10 @@ vm.mostFrequent = { header: "Most Frequent", - get: eventService.getAll, + get: stackService.getRollups, options: { limit: 15, - mode: "stack_frequent", + sort: "-total", }, source: vm._source + ".Events", }; diff --git a/src/Exceptionless.Web/ClientApp.angular/app/new-controller.js b/src/Exceptionless.Web/ClientApp.angular/app/new-controller.js index e1dbad0269..f6d3142bfc 100644 --- a/src/Exceptionless.Web/ClientApp.angular/app/new-controller.js +++ b/src/Exceptionless.Web/ClientApp.angular/app/new-controller.js @@ -108,6 +108,23 @@ return organizationService.getAll().then(onSuccess); } + function getNewStacks(options) { + return stackService.getRollups(options, function (mergedOptions) { + var range = filterService.getTimeRange(); + if (!range.start && !range.end) { + return mergedOptions; + } + + var start = (range.start || moment(filterService.getOldestPossibleEventDate())).utc().format(); + var end = (range.end || moment()).utc().format(); + var firstOccurrenceFilter = 'first_occurrence:["' + start + '" TO "' + end + '"]'; + mergedOptions.filter = mergedOptions.filter + ? firstOccurrenceFilter + " (" + mergedOptions.filter + ")" + : firstOccurrenceFilter; + return mergedOptions; + }); + } + this.$onInit = function $onInit() { vm._organizations = []; vm._source = "app.New"; @@ -212,10 +229,10 @@ vm.newest = { header: "New Stacks", - get: eventService.getAll, + get: getNewStacks, options: { limit: 15, - mode: "stack_new", + sort: "-first_occurrence", }, source: vm._source + ".Events", }; diff --git a/src/Exceptionless.Web/ClientApp.angular/app/users-controller.js b/src/Exceptionless.Web/ClientApp.angular/app/users-controller.js index f6ed12eaf3..72a00a992f 100644 --- a/src/Exceptionless.Web/ClientApp.angular/app/users-controller.js +++ b/src/Exceptionless.Web/ClientApp.angular/app/users-controller.js @@ -212,10 +212,10 @@ vm.mostUsers = { header: "Most Users", - get: eventService.getAll, + get: stackService.getRollups, options: { limit: 15, - mode: "stack_users", + sort: "-users", }, source: vm._source + ".Events", }; diff --git a/src/Exceptionless.Web/ClientApp.angular/components/stack/stack-service.js b/src/Exceptionless.Web/ClientApp.angular/components/stack/stack-service.js index ce22340668..91bb8403c6 100644 --- a/src/Exceptionless.Web/ClientApp.angular/components/stack/stack-service.js +++ b/src/Exceptionless.Web/ClientApp.angular/components/stack/stack-service.js @@ -33,58 +33,25 @@ return Restangular.one("stacks", id).get(); } - function getFrequent(options) { - var mergedOptions = filterService.apply(options); - var organization = filterService.getOrganizationId(); - if (organization) { - return Restangular.one("organizations", organization) - .one("stacks") - .all("frequent") - .getList(mergedOptions); - } - - var project = filterService.getProjectId(); - if (project) { - return Restangular.one("projects", project).one("stacks").all("frequent").getList(mergedOptions); - } - - return Restangular.one("stacks").all("frequent").getList(mergedOptions); - } - - function getUsers(options) { - var mergedOptions = filterService.apply(options); - var organization = filterService.getOrganizationId(); - if (organization) { - return Restangular.one("organizations", organization) - .one("stacks") - .all("users") - .getList(mergedOptions); - } - - var project = filterService.getProjectId(); - if (project) { - return Restangular.one("projects", project).one("stacks").all("users").getList(mergedOptions); - } - - return Restangular.one("stacks").all("users").getList(mergedOptions); - } - - function getNew(options) { - var mergedOptions = filterService.apply(options); + function getRollups(options, optionsCallback) { + optionsCallback = angular.isFunction(optionsCallback) + ? optionsCallback + : function (o) { + return o; + }; + var mergedOptions = optionsCallback(filterService.apply(options)); + mergedOptions.mode = "stack"; var organization = filterService.getOrganizationId(); if (organization) { - return Restangular.one("organizations", organization) - .one("stacks") - .all("new") - .getList(mergedOptions); + return Restangular.one("organizations", organization).all("events").getList(mergedOptions); } var project = filterService.getProjectId(); if (project) { - return Restangular.one("projects", project).one("stacks").all("new").getList(mergedOptions); + return Restangular.one("projects", project).all("events").getList(mergedOptions); } - return Restangular.one("stacks").all("new").getList(mergedOptions); + return Restangular.all("events").getList(mergedOptions); } function markCritical(id) { @@ -122,9 +89,7 @@ changeStatus: changeStatus, getAll: getAll, getById: getById, - getFrequent: getFrequent, - getUsers: getUsers, - getNew: getNew, + getRollups: getRollups, markCritical: markCritical, markNotCritical: markNotCritical, markFixed: markFixed, diff --git a/src/Exceptionless.Web/ClientApp/e2e/tests/list-query-cache.e2e.ts b/src/Exceptionless.Web/ClientApp/e2e/tests/list-query-cache.e2e.ts index 797ae09030..5a1ff6937f 100644 --- a/src/Exceptionless.Web/ClientApp/e2e/tests/list-query-cache.e2e.ts +++ b/src/Exceptionless.Web/ClientApp/e2e/tests/list-query-cache.e2e.ts @@ -133,7 +133,7 @@ function recordListRequest(counts: RequestCounts, request: Request): void { const isStats = url.pathname.endsWith('/count'); const mode = url.searchParams.get('mode'); - if (mode === 'stack_frequent') { + if (mode === 'stack') { counts[isStats ? 'stackStats' : 'stackList']++; } else if (isStats || mode === 'summary') { counts[isStats ? 'eventStats' : 'eventList']++; diff --git a/src/Exceptionless.Web/ClientApp/e2e/tests/stack-effects-chaos.e2e.ts b/src/Exceptionless.Web/ClientApp/e2e/tests/stack-effects-chaos.e2e.ts index 3f8a565737..b117b0f28c 100644 --- a/src/Exceptionless.Web/ClientApp/e2e/tests/stack-effects-chaos.e2e.ts +++ b/src/Exceptionless.Web/ClientApp/e2e/tests/stack-effects-chaos.e2e.ts @@ -131,9 +131,11 @@ test('stack effects stay bounded through background, paging, and navigation chao await measureAction(diagnostics, 'paging', async () => { for (let index = 0; index < 4; index++) { await page.getByRole('button', { name: 'Go to next page' }).click(); - await expect(page).toHaveURL(/(?:\?|&)page=2(?:&|$)/); + await expect(page).toHaveURL(/(?:\?|&)after=[^&]+(?:&|$)/); + await expect(page).not.toHaveURL(/(?:\?|&)page=/); await page.getByRole('button', { name: 'Go to previous page' }).click(); - await expect(page).not.toHaveURL(/(?:\?|&)page=2(?:&|$)/); + await expect(page).not.toHaveURL(/(?:\?|&)(?:before|after)=/); + await expect(page).not.toHaveURL(/(?:\?|&)page=/); } }); expect(actionSample(diagnostics, 'paging').listRequests).toBe(1); @@ -268,7 +270,7 @@ function createChaosEvent(appUrl: string, run: string, index: number): { event: function isStackListRequest(request: Request, organizationId: string): boolean { const url = new URL(request.url()); - return url.pathname === `/api/v2/organizations/${organizationId}/events` && url.searchParams.get('mode') === 'stack_frequent'; + return url.pathname === `/api/v2/organizations/${organizationId}/events` && url.searchParams.get('mode') === 'stack'; } function isStackListResponse(response: Response, organizationId: string): boolean { @@ -322,7 +324,7 @@ function recordRequest(diagnostics: RuntimeDiagnostics, request: Request, organi } const url = new URL(request.url()); - if (url.pathname === `/api/v2/organizations/${organizationId}/events/count` && url.searchParams.get('mode') === 'stack_frequent') { + if (url.pathname === `/api/v2/organizations/${organizationId}/events/count` && url.searchParams.get('mode') === 'stack') { diagnostics.countRequests++; } } diff --git a/src/Exceptionless.Web/ClientApp/e2e/tests/stack-triage.e2e.ts b/src/Exceptionless.Web/ClientApp/e2e/tests/stack-triage.e2e.ts index efbebab3d9..477694bd76 100644 --- a/src/Exceptionless.Web/ClientApp/e2e/tests/stack-triage.e2e.ts +++ b/src/Exceptionless.Web/ClientApp/e2e/tests/stack-triage.e2e.ts @@ -1,5 +1,40 @@ import { expect, test } from '../fixtures/e2e-test'; import { ExceptionlessE2EJourney } from '../support/exceptionless-journey'; +import { createRepresentativeEvent } from '../support/synthetic-event'; + +test('project stack management uses cursor pagination without page numbers @signup', async ({ e2eApi, e2eScenario, page }) => { + const events = Array.from({ length: 6 }, (_, index) => { + const referenceId = `pw-stack-management-${e2eScenario.run}-${index}`; + const event = createRepresentativeEvent({ + appUrl: e2eApi.environment.appUrl, + message: `Project stack management ${e2eScenario.run} ${index}`, + referenceId, + runId: e2eScenario.run + }); + const simpleError = (event.data as Record)['@simple_error'] as Record; + simpleError.type = `ProjectStackManagementException${index}`; + simpleError.stack_trace = `Error: ${referenceId}\n at stack-management-${index}.ts:${index + 1}:1`; + return { event, referenceId }; + }); + + await Promise.all(events.map(({ event }) => e2eApi.submitEvent(e2eScenario.projectId, e2eScenario.projectToken, event))); + await Promise.all(events.map(({ referenceId }) => e2eApi.pollForEventByReference(e2eScenario.userToken, e2eScenario.projectId, referenceId))); + + const listResponse = page.waitForResponse((response) => new URL(response.url()).pathname === `/api/v2/projects/${e2eScenario.projectId}/stacks`); + await page.goto(`/next/project/${e2eScenario.projectId}/stacks?filter=status%3Aopen&limit=5`); + expect((await listResponse).ok()).toBe(true); + await expect(page.getByText('Manage project stacks, including restoring ignored or discarded stacks')).toBeVisible(); + await expect(page.locator('tbody tr:visible')).toHaveCount(5); + + await page.getByRole('button', { name: 'Go to next page' }).click(); + await expect(page).toHaveURL(/(?:\?|&)after=[^&]+(?:&|$)/); + await expect(page).not.toHaveURL(/(?:\?|&)page=/); + await expect(page.locator('tbody tr:visible').first()).toBeVisible(); + + await page.getByRole('button', { name: 'Go to previous page' }).click(); + await expect(page).not.toHaveURL(/(?:\?|&)(?:before|after|page)=/); + await expect(page.locator('tbody tr:visible')).toHaveCount(5); +}); test('new user can mark an open stack fixed from event details @signup', async ({ e2eApi, e2eScenario, page }) => { const journey = ExceptionlessE2EJourney.fromScenario(page, e2eApi, e2eScenario); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts index 16bcce87b3..2554418dfa 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts @@ -13,18 +13,21 @@ import type { PersistentEvent } from './models'; export interface OrganizationEventNotificationRefresher { cancel: () => void; - schedule: (organizationId?: string, refreshImmediately?: boolean) => void; + schedule: (organizationId?: string, refreshImmediately?: boolean, includeStackLists?: boolean) => void; } export function createOrganizationEventNotificationRefresher(queryClient: QueryClient): OrganizationEventNotificationRefresher { const pendingOrganizationIds = new SvelteSet(); + let pendingStackListRefresh = false; let trailingRefresh: ReturnType | undefined; const refresh = () => { const organizationIds = [...pendingOrganizationIds]; + const includeStackLists = pendingStackListRefresh; void queryClient.invalidateQueries({ predicate: (query) => isOrganizationEventDashboardQueryKey(query.queryKey) && + (includeStackLists || !isOrganizationStackListQueryKey(query.queryKey)) && (organizationIds.includes(undefined) || organizationIds.includes(query.queryKey[2] as string)), queryKey: queryKeys.type, refetchType: 'active' @@ -34,13 +37,15 @@ export function createOrganizationEventNotificationRefresher(queryClient: QueryC return { cancel: () => { pendingOrganizationIds.clear(); + pendingStackListRefresh = false; if (trailingRefresh !== undefined) { clearTimeout(trailingRefresh); trailingRefresh = undefined; } }, - schedule: (organizationId?: string, refreshImmediately = true) => { + schedule: (organizationId?: string, refreshImmediately = true, includeStackLists = false) => { pendingOrganizationIds.add(organizationId); + pendingStackListRefresh ||= includeStackLists; if (trailingRefresh !== undefined) { return; } @@ -53,6 +58,7 @@ export function createOrganizationEventNotificationRefresher(queryClient: QueryC trailingRefresh = undefined; refresh(); pendingOrganizationIds.clear(); + pendingStackListRefresh = false; }, ORGANIZATION_EVENT_NOTIFICATION_THROTTLE_MS); } }; @@ -163,7 +169,7 @@ export interface GetEventsByReferenceRequest { }; } -export type GetEventsMode = 'stack_frequent' | 'stack_new' | 'stack_recent' | 'stack_users' | 'summary' | null; +export type GetEventsMode = 'stack' | 'summary' | null; export interface GetEventsParams { after?: string; @@ -183,7 +189,7 @@ export interface GetOrganizationCountRequest { params?: { aggregations?: string; filter?: string; - mode?: GetEventsMode; + mode?: 'stack'; offset?: string; time?: string; }; @@ -216,7 +222,7 @@ export interface GetProjectCountRequest { params?: { aggregations?: string; filter?: string; - mode?: 'stack_new'; + mode?: 'stack'; offset?: string; time?: string; }; @@ -246,7 +252,6 @@ export interface GetStackCountRequest { params?: { aggregations?: string; filter?: string; - mode?: 'stack_new'; offset?: string; time?: string; }; @@ -434,8 +439,8 @@ export function getOrganizationCountQuery(request: GetOrganizationCountRequest) }); } -export function getOrganizationEventsQuery(request: GetOrganizationEventsRequest) { - return createQuery[]>, ProblemDetails>(() => { +export function getOrganizationEventsQuery>(request: GetOrganizationEventsRequest) { + return createQuery, ProblemDetails>(() => { const organizationId = request.route.organizationId; const params = request.params ? { @@ -448,7 +453,7 @@ export function getOrganizationEventsQuery(request: GetOrganizationEventsRequest placeholderData: keepPreviousData, queryFn: async () => { const client = useFetchClient(); - return await client.getJSON[]>(`organizations/${organizationId}/events`, { + return await client.getJSON(`organizations/${organizationId}/events`, { params: params as Record }); }, @@ -642,3 +647,7 @@ function isOrganizationEventDashboardQueryKey(queryKey: readonly unknown[]): boo function isOrganizationEventsQueryKey(queryKey: readonly unknown[]): boolean { return queryKey[0] === queryKeys.type[0] && queryKey[1] === 'organizations' && queryKey[3] === 'events'; } + +function isOrganizationStackListQueryKey(queryKey: readonly unknown[]): boolean { + return isOrganizationEventsQueryKey(queryKey) && (queryKey[4] as GetEventsParams | undefined)?.mode === 'stack'; +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.test.ts index d6e7407430..6501bfc387 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.test.ts @@ -87,6 +87,8 @@ describe('createOrganizationEventNotificationRefresher', () => { expect(firstInvalidation?.refetchType).toBe('active'); expect(firstInvalidation?.predicate?.({ queryKey: queryKeys.organizationsEvents('organization-id') } as never)).toBe(true); expect(firstInvalidation?.predicate?.({ queryKey: queryKeys.organizationsCount('organization-id') } as never)).toBe(true); + expect(firstInvalidation?.predicate?.({ queryKey: queryKeys.organizationsEvents('organization-id', { mode: 'stack' }) } as never)).toBe(false); + expect(firstInvalidation?.predicate?.({ queryKey: queryKeys.organizationsCount('organization-id', { mode: 'stack' }) } as never)).toBe(true); expect(firstInvalidation?.predicate?.({ queryKey: queryKeys.organizationsEvents('other-organization-id') } as never)).toBe(false); expect(firstInvalidation?.predicate?.({ queryKey: queryKeys.id('event-id') } as never)).toBe(false); @@ -105,6 +107,23 @@ describe('createOrganizationEventNotificationRefresher', () => { await vi.advanceTimersByTimeAsync(ORGANIZATION_EVENT_NOTIFICATION_THROTTLE_MS); expect(invalidateSpy).toHaveBeenCalledTimes(3); }); + + it('includes stack-mode lists only when scheduled by a stack notification', async () => { + // Arrange + vi.useFakeTimers(); + const queryClient = new QueryClient(); + const invalidateSpy = vi.spyOn(queryClient, 'invalidateQueries').mockImplementation(async () => {}); + const refresher = createOrganizationEventNotificationRefresher(queryClient); + + // Act + refresher.schedule('organization-id', true, true); + + // Assert + const invalidation = invalidateSpy.mock.calls[0]?.[0]; + expect(invalidation?.predicate?.({ queryKey: queryKeys.organizationsEvents('organization-id', { mode: 'stack' }) } as never)).toBe(true); + + refresher.cancel(); + }); }); describe('createEventWithNavigationQueryOptions', () => { diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-data-table.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-data-table.svelte index 03564d2ad5..ccdcf65329 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-data-table.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-data-table.svelte @@ -1,11 +1,11 @@ -