feat: limits

api-contracts/openapi/components/schemas/_index.yaml

@@ -36,6 +36,12 @@ TenantMemberList:
   $ref: "./tenant.yaml#/TenantMemberList"
 TenantMemberRole:
   $ref: "./tenant.yaml#/TenantMemberRole"
+TenantResource:
+  $ref: "./tenant.yaml#/TenantResource"
+TenantResourceLimit:
+  $ref: "./tenant.yaml#/TenantResourceLimit"
+TenantResourcePolicy:
+  $ref: "./tenant.yaml#/TenantResourcePolicy"
 CreateTenantInviteRequest:
   $ref: "./tenant.yaml#/CreateTenantInviteRequest"
 UpdateTenantInviteRequest:

api-contracts/openapi/components/schemas/tenant.yaml

@@ -33,6 +33,9 @@ TenantAlertingSettings:
     enableExpiringTokenAlerts:
       type: boolean
       description: Whether to enable alerts when tokens are approaching expiration.
+    enableTenantResourceLimitAlerts:
+      type: boolean
+      description: Whether to enable alerts when tenant resources are approaching limits.
     maxAlertingFrequency:
       type: string
       description: The max frequency at which to alert.
@@ -79,13 +82,67 @@ UpdateTenantRequest:
     enableExpiringTokenAlerts:
       type: boolean
       description: Whether to enable alerts when tokens are approaching expiration.
+    enableTenantResourceLimitAlerts:
+      type: boolean
+      description: Whether to enable alerts when tenant resources are approaching limits.
     maxAlertingFrequency:
       type: string
       description: The max frequency at which to alert.
       x-oapi-codegen-extra-tags:
         validate: "omitnil,duration"
   type: object
 
+TenantResource:
+  enum:
+    - "WORKER"
+    - "EVENT"
+    - "WORKFLOW_RUN"
+    - "CRON"
+    - "SCHEDULE"
+  type: string
+
+TenantResourceLimit:
+  properties:
+    metadata:
+      $ref: "./metadata.yaml#/APIResourceMeta"
+    resource:
+      $ref: "#/TenantResource"
+      description: The resource associated with this limit.
+    limitValue:
+      type: integer
+      description: The limit associated with this limit.
+    alarmValue:
+      type: integer
+      description: The alarm value associated with this limit to warn of approaching limit value.
+    value:
+      type: integer
+      description: The current value associated with this limit.
+    window:
+      type: string
+      description: The meter window for the limit. (i.e. 1 day, 1 week, 1 month)
+    lastRefill:
+      type: string
+      description: The last time the limit was refilled.
+      format: date-time
+  required:
+    - metadata
+    - tenantId
+    - resource
+    - limitValue
+    - value
+  type: object
+
+TenantResourcePolicy:
+  properties:
+    limits:
+      type: array
+      items:
+        $ref: "#/TenantResourceLimit"
+      description: A list of resource limits for the tenant.
+  required:
+    - limits
+  type: object
+
 TenantMember:
   properties:
     metadata:

api-contracts/openapi/openapi.yaml

@@ -56,6 +56,8 @@ paths:
     $ref: "./paths/ingestors/ingestors.yaml#/snsIntegration"
   /api/v1/tenants/{tenant}/alerting-email-groups:
     $ref: "./paths/tenant/tenant.yaml#/tenantAlertEmailGroups"
+  /api/v1/tenants/{tenant}/resource-policy:
+    $ref: "./paths/tenant/tenant.yaml#/tenantResourcePolicy"
   /api/v1/alerting-email-groups/{alert-email-group}:
     $ref: "./paths/tenant/tenant.yaml#/alertEmailGroup"
   /api/v1/sns/{sns}:

api-contracts/openapi/paths/event/event.yaml

@@ -216,6 +216,12 @@ replayEvents:
             schema:
               $ref: "../../components/schemas/_index.yaml#/APIErrors"
         description: Forbidden
+      "429":
+        content:
+          application/json:
+            schema:
+              $ref: "../../components/schemas/_index.yaml#/APIErrors"
+        description: Resource limit exceeded
     summary: Replay events
     tags:
       - Event

api-contracts/openapi/paths/tenant/tenant.yaml

@@ -269,6 +269,45 @@ tenantAlertEmailGroups:
     summary: List tenant alert email groups
     tags:
       - Tenant
+
+tenantResourcePolicy:
+  get:
+    x-resources: ["tenant"]
+    description: Gets the resource policy for a tenant
+    operationId: tenant-resource-policy:get
+    parameters:
+      - description: The tenant id
+        in: path
+        name: tenant
+        required: true
+        schema:
+          type: string
+          format: uuid
+          minLength: 36
+          maxLength: 36
+    responses:
+      "200":
+        content:
+          application/json:
+            schema:
+              $ref: "../../components/schemas/_index.yaml#/TenantResourcePolicy"
+        description: Successfully retrieved the tenant resource policy
+      "400":
+        content:
+          application/json:
+            schema:
+              $ref: "../../components/schemas/_index.yaml#/APIErrors"
+        description: A malformed or bad request
+      "403":
+        content:
+          application/json:
+            schema:
+              $ref: "../../components/schemas/_index.yaml#/APIError"
+        description: Forbidden
+    summary: Create tenant alert email group
+    tags:
+      - Tenant
+
 invites:
   post:
     x-resources: ["tenant"]

api-contracts/openapi/paths/workflow/workflow.yaml

@@ -258,6 +258,12 @@ triggerWorkflow:
             schema:
               $ref: "../../components/schemas/_index.yaml#/APIErrors"
         description: A malformed or bad request
+      "429":
+        content:
+          application/json:
+            schema:
+              $ref: "../../components/schemas/_index.yaml#/APIErrors"
+        description: Resource limit exceeded
       "403":
         content:
           application/json:

api/v1/server/handlers/events/replay.go

@@ -4,8 +4,10 @@ import (
 	"github.com/hashicorp/go-multierror"
 	"github.com/labstack/echo/v4"
 
+	"github.com/hatchet-dev/hatchet/api/v1/server/oas/apierrors"
 	"github.com/hatchet-dev/hatchet/api/v1/server/oas/gen"
 	"github.com/hatchet-dev/hatchet/api/v1/server/oas/transformers"
+	"github.com/hatchet-dev/hatchet/internal/repository/metered"
 	"github.com/hatchet-dev/hatchet/internal/repository/prisma/db"
 	"github.com/hatchet-dev/hatchet/internal/repository/prisma/sqlchelpers"
 )
@@ -34,6 +36,12 @@ func (t *EventService) EventUpdateReplay(ctx echo.Context, request gen.EventUpda
 
 		newEvent, err := t.config.Ingestor.IngestReplayedEvent(ctx.Request().Context(), tenant.ID, event)
 
+		if err == metered.ErrResourceExhausted {
+			return gen.EventUpdateReplay429JSONResponse(
+				apierrors.NewAPIErrors("Event limit exceeded"),
+			), nil
+		}
+
 		if err != nil {
 			allErrs = multierror.Append(allErrs, err)
 		}

api/v1/server/handlers/tenants/create.go

@@ -1,6 +1,7 @@
 package tenants
 
 import (
+	"context"
 	"errors"
 
 	"github.com/labstack/echo/v4"
@@ -48,6 +49,12 @@ func (t *TenantService) TenantCreate(ctx echo.Context, request gen.TenantCreateR
 		return nil, err
 	}
 
+	err = t.config.EntitlementRepository.TenantLimit().CreateTenantDefaultLimits(context.Background(), tenant.ID)
+
+	if err != nil {
+		return nil, err
+	}
+
 	// add the user as an owner of the tenant
 	_, err = t.config.APIRepository.Tenant().CreateTenantMember(tenant.ID, &repository.CreateTenantMemberOpts{
 		UserId: user.ID,

api/v1/server/handlers/tenants/get_resource_policy.go

@@ -0,0 +1,25 @@
+package tenants
+
+import (
+	"context"
+
+	"github.com/labstack/echo/v4"
+
+	"github.com/hatchet-dev/hatchet/api/v1/server/oas/gen"
+	"github.com/hatchet-dev/hatchet/api/v1/server/oas/transformers"
+	"github.com/hatchet-dev/hatchet/internal/repository/prisma/db"
+)
+
+func (t *TenantService) TenantResourcePolicyGet(ctx echo.Context, request gen.TenantResourcePolicyGetRequestObject) (gen.TenantResourcePolicyGetResponseObject, error) {
+	tenant := ctx.Get("tenant").(*db.TenantModel)
+
+	limits, err := t.config.EntitlementRepository.TenantLimit().GetLimits(context.Background(), tenant.ID)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return gen.TenantResourcePolicyGet200JSONResponse(
+		*transformers.ToTenantResourcePolicy(limits),
+	), nil
+}

api/v1/server/handlers/tenants/update.go

@@ -43,14 +43,16 @@ func (t *TenantService) TenantUpdate(ctx echo.Context, request gen.TenantUpdateR
 
 	if request.Body.MaxAlertingFrequency != nil ||
 		request.Body.EnableExpiringTokenAlerts != nil ||
+		request.Body.EnableTenantResourceLimitAlerts != nil ||
 		request.Body.EnableWorkflowRunFailureAlerts != nil {
 
 		_, err = t.config.APIRepository.TenantAlertingSettings().UpsertTenantAlertingSettings(
 			tenant.ID,
 			&repository.UpsertTenantAlertingSettingsOpts{
-				MaxFrequency:                   request.Body.MaxAlertingFrequency,
-				EnableExpiringTokenAlerts:      request.Body.EnableExpiringTokenAlerts,
-				EnableWorkflowRunFailureAlerts: request.Body.EnableWorkflowRunFailureAlerts,
+				MaxFrequency:                    request.Body.MaxAlertingFrequency,
+				EnableExpiringTokenAlerts:       request.Body.EnableExpiringTokenAlerts,
+				EnableWorkflowRunFailureAlerts:  request.Body.EnableWorkflowRunFailureAlerts,
+				EnableTenantResourceLimitAlerts: request.Body.EnableTenantResourceLimitAlerts,
 			},
 		)
 

api/v1/server/handlers/workflows/trigger.go

@@ -12,6 +12,7 @@ import (
 	"github.com/hatchet-dev/hatchet/api/v1/server/oas/transformers"
 	"github.com/hatchet-dev/hatchet/internal/msgqueue"
 	"github.com/hatchet-dev/hatchet/internal/repository"
+	"github.com/hatchet-dev/hatchet/internal/repository/metered"
 	"github.com/hatchet-dev/hatchet/internal/repository/prisma/db"
 	"github.com/hatchet-dev/hatchet/internal/services/shared/tasktypes"
 )
@@ -83,6 +84,12 @@ func (t *WorkflowService) WorkflowRunCreate(ctx echo.Context, request gen.Workfl
 
 	workflowRun, err := t.config.APIRepository.WorkflowRun().CreateNewWorkflowRun(ctx.Request().Context(), tenant.ID, createOpts)
 
+	if err == metered.ErrResourceExhausted {
+		return gen.WorkflowRunCreate429JSONResponse(
+			apierrors.NewAPIErrors("Workflow Run limit exceeded"),
+		), nil
+	}
+
 	if err != nil {
 		return nil, fmt.Errorf("could not create workflow run: %w", err)
 	}