Columns
| Column | Type | Size | Nulls | Auto | Default | Children | Parents | Comments |
|---|---|---|---|---|---|---|---|---|
| url | text | 2147483647 | √ | null |
|
|
{“what”:“Full URL of the page, e.g., https://example.com/a.”,“why”:“Uniquely identifies the source; needed for deduplication and linking.”,“by”:“fleet-annotator-worker”,“at”:“2026-09-23T04:05:41.226940506+00:00”} |
|
| domain | text | 2147483647 | √ | null |
|
|
{“what”:“Domain part of the URL, e.g., example.com.”,“why”:“Groups pages by site for domain-level analysis.”,“by”:“fleet-annotator-worker”,“at”:“2026-09-23T04:05:41.252935807+00:00”} |
|
| title | text | 2147483647 | √ | null |
|
|
{“what”:“Page title as extracted from HTML.”,“why”:“Provides a short human-readable label for the content.”,“by”:“fleet-annotator-worker”,“at”:“2026-09-23T04:05:41.430977641+00:00”} |
|
| body | text | 2147483647 | √ | null |
|
|
{“what”:“Main text content of the page, stripped of HTML.”,“why”:“The actual corpus text used for analysis and model training.”,“by”:“fleet-annotator-worker”,“at”:“2026-09-23T04:05:41.447170920+00:00”} |
|
| language | text | 2147483647 | √ | null |
|
|
{“what”:“ISO 639-1 language code, e.g., ‘en’.”,“why”:“Filters and processes content per language.”,“by”:“fleet-annotator-worker”,“at”:“2026-09-23T04:05:41.478610010+00:00”} |
|
| category | text | 2147483647 | √ | null |
|
|
{“what”:“Topic category assigned, e.g., ‘tech’.”,“why”:“Allows filtering by topic for evaluation.”,“by”:“fleet-annotator-worker”,“at”:“2026-09-23T04:05:41.684226537+00:00”} |
|
| quality_score | float4 | 8,8 | √ | null |
|
|
{“what”:“Quality score from 0.0 to 1.0, higher is better.”,“why”:“Filters out low-quality pages before training.”,“by”:“fleet-annotator-worker”,“at”:“2026-09-23T04:05:41.706460389+00:00”} |
|
| content_hash | text | 2147483647 | √ | null |
|
|
{“what”:“SHA-256 hash of the body text.”,“why”:“Detects duplicate content and verifies integrity.”,“by”:“fleet-annotator-worker”,“at”:“2026-09-23T04:05:41.735964231+00:00”} |
|
| source_segment | text | 2147483647 | √ | null |
|
|
{“what”:“Identifier of the source batch or partition.”,“why”:“Tracks provenance and enables partial reprocessing.”,“by”:“fleet-annotator-worker”,“at”:“2026-09-23T04:05:41.753873184+00:00”} |
|
| crawl_date | text | 2147483647 | √ | null |
|
|
{“what”:“Crawl date in YYYY-MM-DD format.”,“why”:“Supports time-based analysis and freshness checks.”,“by”:“fleet-annotator-worker”,“at”:“2026-09-23T04:05:41.780472322+00:00”} |