| url |
varchar |
2147483647 |
√ |
|
null |
|
|
{“what”:“Full URL of the web page, e.g., https://example.com/page.”,“why”:“Unique identifier for the document; needed to reference or fetch the source.”,“by”:“fleet-annotator-worker”,“at”:“2026-09-23T00:02:24.531230766+00:00”} |
| domain |
varchar |
2147483647 |
√ |
|
null |
|
|
{“what”:“Domain name extracted from URL, e.g., example.com.”,“why”:“Enables grouping or filtering by site without parsing full URLs.”,“by”:“fleet-annotator-worker”,“at”:“2026-09-23T00:02:24.576745674+00:00”} |
| title |
varchar |
2147483647 |
√ |
|
null |
|
|
{“what”:“Title text of the web page as in HTML .”,“why”:“Provides a concise human-readable label for display or search results.”,“by”:“fleet-annotator-worker”,“at”:“2026-09-23T00:02:24.597279880+00:00”}</p></td>
</tr>
<tr>
<td><span id="body">body</span></td>
<td>varchar</td>
<td>2147483647</td>
<td title='nullable'>√</td>
<td title=''></td>
<td>null</td>
<td>
<table border='0' cellspacing='0' cellpadding='0'>
</table>
</td>
<td>
<table border='0' cellspacing='0' cellpadding='0'>
</table>
</td>
<td><p>{“what”:“Full text content of the page, stripped of HTML tags.”,“why”:“Primary text for NLP tasks; without it there is no content to analyze.”,“by”:“fleet-annotator-worker”,“at”:“2026-09-23T00:02:24.622097145+00:00”}</p></td>
</tr>
<tr>
<td><span id="language">language</span></td>
<td>varchar</td>
<td>2147483647</td>
<td title='nullable'>√</td>
<td title=''></td>
<td>null</td>
<td>
<table border='0' cellspacing='0' cellpadding='0'>
</table>
</td>
<td>
<table border='0' cellspacing='0' cellpadding='0'>
</table>
</td>
<td><p>{“what”:“ISO 639-1 language code (e.g., ‘en’) of the body text.”,“why”:“Needed for language-specific processing or filtering by language.”,“by”:“fleet-annotator-worker”,“at”:“2026-09-23T00:02:24.642955999+00:00”}</p></td>
</tr>
<tr>
<td><span id="category">category</span></td>
<td>varchar</td>
<td>2147483647</td>
<td title='nullable'>√</td>
<td title=''></td>
<td>null</td>
<td>
<table border='0' cellspacing='0' cellpadding='0'>
</table>
</td>
<td>
<table border='0' cellspacing='0' cellpadding='0'>
</table>
</td>
<td><p>{“what”:“Topic category label assigned to the page (e.g., ‘sports’).”,“why”:“Enables classification or filtering by topic for targeted analysis.”,“by”:“fleet-annotator-worker”,“at”:“2026-09-23T00:02:24.672128887+00:00”}</p></td>
</tr>
<tr>
<td><span id="quality_score">quality_score</span></td>
<td>float4</td>
<td>8,8</td>
<td title='nullable'>√</td>
<td title=''></td>
<td>null</td>
<td>
<table border='0' cellspacing='0' cellpadding='0'>
</table>
</td>
<td>
<table border='0' cellspacing='0' cellpadding='0'>
</table>
</td>
<td><p>{“what”:“Real number score (higher is better) indicating content quality.”,“why”:“Used to rank or filter low-quality pages from the corpus.”,“by”:“fleet-annotator-worker”,“at”:“2026-09-23T00:02:24.701082240+00:00”}</p></td>
</tr>
<tr>
<td><span id="content_hash">content_hash</span></td>
<td>varchar</td>
<td>2147483647</td>
<td title='nullable'>√</td>
<td title=''></td>
<td>null</td>
<td>
<table border='0' cellspacing='0' cellpadding='0'>
</table>
</td>
<td>
<table border='0' cellspacing='0' cellpadding='0'>
</table>
</td>
<td><p>{“what”:“Hash (e.g., SHA-256) of the body content for deduplication.”,“why”:“Detects duplicate pages without comparing full text, saving storage.”,“by”:“fleet-annotator-worker”,“at”:“2026-09-23T00:02:24.721364431+00:00”}</p></td>
</tr>
<tr>
<td><span id="source_segment">source_segment</span></td>
<td>varchar</td>
<td>2147483647</td>
<td title='nullable'>√</td>
<td title=''></td>
<td>null</td>
<td>
<table border='0' cellspacing='0' cellpadding='0'>
</table>
</td>
<td>
<table border='0' cellspacing='0' cellpadding='0'>
</table>
</td>
<td><p>{“what”:“Identifier of the crawl batch or partition this page came from.”,“why”:“Tracks provenance and allows selective processing of specific segments.”,“by”:“fleet-annotator-worker”,“at”:“2026-09-23T00:02:24.753025082+00:00”}</p></td>
</tr>
<tr>
<td><span id="crawl_date">crawl_date</span></td>
<td>varchar</td>
<td>2147483647</td>
<td title='nullable'>√</td>
<td title=''></td>
<td>null</td>
<td>
<table border='0' cellspacing='0' cellpadding='0'>
</table>
</td>
<td>
<table border='0' cellspacing='0' cellpadding='0'>
</table>
</td>
<td><p>{“what”:“Date of crawl in YYYY-MM-DD format.”,“why”:“Enables time-based filtering or freshness checks of the corpus.”,“by”:“fleet-annotator-worker”,“at”:“2026-09-23T00:02:24.778494876+00:00”}</p></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="box box-primary">
<div class="box-header with-border">
<i class="fa fa-code-fork"></i>
<h3 id="Relationships" class="box-title">Relationships</h3>
<div class="box-tools pull-right">
<button type="button" class="btn btn-box-tool" data-widget="collapse"><i class="fa fa-minus"></i></button>
<button type="button" class="btn btn-box-tool" data-widget="remove"><i class="fa fa-times"></i></button>
</div>
</div>
<div class="box-body">
<div class="nav-tabs-custom"><!-- Tabs within a box -->
<h5>Close relationships within degrees of separation</h5>
<ul class="nav nav-tabs pull-left ui-sortable-handle">
<li class="active"><a href="#oneDegreeImg-chart" data-toggle="tab" aria-expanded="true">One</a></li>
</ul>
<div class="tab-content no-padding">
<div class="chart tab-pane active" id="oneDegreeImg-chart" style="position: relative; overflow-x:auto;">
<a name='diagram'><object type="image/svg+xml" id="oneDegreeImg" data="../diagrams/tables/ds_parquetrhythm_web_corpus_v1.1degree.svg" class="diagram" border="0" align="left"></object></a>
</div>
</div>
</div>
</div><!-- /.box-body -->
</div>
</section>
<script>
var config = {
pagination: true
}
</script>
</div>
<!-- /.content-wrapper -->
<footer class="main-footer">
<div>
<div class="pull-right hidden-xs">
<a href="https://github.com/schemaspy/schemaspy" title="GitHub for SchemaSpy"><i class="fa fa-github-square fa-2x"></i></a>
<a href="http://stackoverflow.com/questions/tagged/schemaspy" title="StackOverflow for SchemaSpy"><i class="fa fa-stack-overflow fa-2x"></i></a>
</div>
<strong>Generated by <a href="http://schemaspy.org/" class="logo-text"><i class="fa fa-database"></i> SchemaSpy 6.2.4</a></strong>
</div>
<!-- /.container -->
</footer>
</div>
<!-- ./wrapper -->
<!-- jQuery 2.2.3 -->
<script src="../bower/admin-lte/plugins/jQuery/jquery-2.2.3.min.js"></script>
<script src="../bower/admin-lte/plugins/jQueryUI/jquery-ui.min.js"></script>
<!-- Bootstrap 3.3.5 -->
<script src="../bower/admin-lte/bootstrap/js/bootstrap.min.js"></script>
<!-- DataTables -->
<script src="../bower/datatables.net/jquery.dataTables.min.js"></script>
<script src="../bower/datatables.net-bs/js/dataTables.bootstrap.min.js"></script>
<script src="../bower/datatables.net-buttons/dataTables.buttons.min.js"></script>
<script src="../bower/datatables.net-buttons-bs/js/buttons.bootstrap.min.js"></script>
<script src="../bower/datatables.net-buttons/buttons.html5.min.js"></script>
<script src="../bower/datatables.net-buttons/buttons.print.min.js"></script>
<script src="../bower/datatables.net-buttons/buttons.colVis.min.js"></script>
<!-- SheetJS -->
<script src="../bower/js-xlsx/xlsx.full.min.js"></script>
<!-- pdfmake -->
<script src="../bower/pdfmake/pdfmake.min.js"></script>
<script src="../bower/pdfmake/vfs_fonts.js"></script>
<!-- SlimScroll -->
<script src="../bower/admin-lte/plugins/slimScroll/jquery.slimscroll.min.js"></script>
<!-- FastClick -->
<script src="../bower/admin-lte/plugins/fastclick/fastclick.js"></script>
<!-- Salvattore -->
<script src="../bower/salvattore/salvattore.min.js"></script>
<!-- AnchorJS -->
<script src="../bower/anchor-js/anchor.min.js"></script>
<!-- CodeMirror -->
<script src="../bower/codemirror/codemirror.js"></script>
<script src="../bower/codemirror/sql.js"></script>
<!-- AdminLTE App -->
<script src="../bower/admin-lte/dist/js/app.min.js"></script>
<script src="table.js"></script>
<script src="../schemaSpy.js"></script>
</body>
</html> |