Search
Teddy Version: LatestInstantiating and utilising the search service client.
Document Store
Teddy provides the ability to index a collection of pages to deliver free-text search and category & tag-based filtering features to your static site. If site.collection.enabled is set to true in the site.json site configuration file for your site, then all pages (for example a collection of blog post pages or similar) located within the directory specified in site.collection.pagesDir, relative to $SITE_BASE/pages, will be indexed and added to a JavaScript-based FlexSearch document store to enable free-text searching, and category & tag-based filtering, of those pages. To disable indexing at the page-level, specify the index key in the page frontmatter and set it to the value false.
For example, the demo TravelBook site is configured in $TEDDY_BASE/sites/travelbook/site.json to index the page name, description, tags and content, thus enabling users of the demo static site to perform free-text search queries across, and category-based filtering of, its travel-related blog posts (recall that, as described in the pages guide, page category keys defined in the page frontmatter will be merged with any page tags during indexing, thus there is no need to add categories to the FlexSearch document store configuration object). The FlexSearch document store configuration object for the demo TravelBook site is as follows.
"collection": {
"enabled": true,
"pagesDir": "blog",
"index": {
"content": true,
"documentStore": {
"document": {
"id": "id",
"tag": "tags",
"index": [
"name",
"description"
],
"store": true
}
}
},
...
}
Please read the site configuration guide for further details regarding the FlexSearch document store configuration object.
Search Service
Teddy provides a convenient search service class that can be instantiated and used by the frontend of your static site to load and query the FlexSearch document store. This search service class can be found in $TEDDY_BASE/system/assets/js/teddy/search.js and, if site.html.inject.systemAssets is set to true in the site.json site configuration, is automatically copied to the public distribution directory when your static site is built - specifically to $SITE_BASE/public/{env}/assets/js/vendors/teddy/{teddy.version}/search.js - and the relevant <script> tag is automatically injected into the generated HTML files. The search service class is as follows.
class Search {
static UNICODE_QUERY_FILTER_REGEX = /[^\p{Letter}\p{Number}_\-\s]/gu;
static LEGACY_QUERY_FILTER_REGEX = /[^a-zA-Z0-9_\-\s\u00c0-\u024f\u4e00-\u9fff\u30a0-\u30ff\u3040-\u309f\uac00-\ud7af\u0370-\u03ff\u0400-\u04ff\u0600-\u06ff\u0e00-\u0e7f\u0900-\u097f\u0980-\u09ff\u0b80-\u0bff\u0a80-\u0aff\u0590-\u05ff\u0f00-\u0fff\u1800-\u18af]/gi;
static WHITESPACE_REGEX = /\s+/g;
static INDEX_KEY_REGEX = /^[a-zA-Z0-9_.-]+$/;
static MAX_MIN_SEARCH_QUERY_LENGTH = 20;
static MAX_SEARCH_QUERY_LENGTH = 200;
static MAX_SEARCH_LIMIT = 100;
static MAX_TAG_COUNT = 30;
static MAX_INDEX_FILE_SIZE_BYTES = 20 * 1024 * 1024;
constructor() {
this.index = null;
this.collectionSize = 0;
this.loaded = false;
}
#assertIndexLoaded() {
if ( !this.loaded || this.index === null ) {
throw new Error('Search index has not been loaded.');
}
}
static sanitizeQuery(query, unicodeProperties = true) {
const normalizedQuery = String(query ?? '')
.slice(0, Search.MAX_SEARCH_QUERY_LENGTH);
const filteredQuery = unicodeProperties ?
// Using Unicode properties.
normalizedQuery.replace(Search.UNICODE_QUERY_FILTER_REGEX, '') :
// Without using Unicode properties (legacy systems).
// Keep the following character sets:
// Alphanumeric a-zA-Z0-9
// Underscore, hyphen and whitespace _\-\s
// Extended Latin (incl. characters for Vietnamese) \u00c0-\u024f
// Chinese characters (CJK unified ideographs) \u4e00-\u9fff
// Japanese Hiragana \u3040-\u309f
// Japanese Katakana \u30a0-\u30ff
// Korean (Hangul syllables) \uac00-\ud7af
// Greek (and Coptic) \u0370-\u03ff
// Cyrillic \u0400-\u04ff
// Arabic \u0600-\u06ff
// Thai \u0e00-\u0e7f
// Hindi (Devanagari) \u0900-\u097f
// Bengali \u0980-\u09ff
// Tamil \u0b80-\u0bff
// Gujarati \u0a80-\u0aff
// Hebrew \u0590-\u05ff
// Tibetan \u0f00-\u0fff
// Mongolian \u1800-\u18af
normalizedQuery.replace(Search.LEGACY_QUERY_FILTER_REGEX, '');
// Clean and standardise whitespace characters.
return filteredQuery
.replace(Search.WHITESPACE_REGEX, ' ')
.trim();
}
static normalizeMinSearchQueryLength(minSearchQueryLength = 2) {
const normalizedMinSearchQueryLength = Math.floor(
Number(minSearchQueryLength) || 0
);
return Math.min(
Math.max(0, normalizedMinSearchQueryLength),
Search.MAX_MIN_SEARCH_QUERY_LENGTH
);
}
static normalizeEnrich(enrich = true) {
return enrich === true;
}
static normalizeTags(tags) {
const normalizedTags = Array.isArray(tags) ? tags : [tags];
return normalizedTags
.filter(function(tag) {
return ['string', 'number'].includes(typeof tag) &&
String(tag).trim().length > 0;
})
.slice(0, Search.MAX_TAG_COUNT);
}
static normalizePagination(offset = 0, limit = 10) {
const normalizedOffset = Math.floor(Number(offset) || 0);
const normalizedLimit = Math.floor(Number(limit) || 0);
return {
offset: Math.max(0, normalizedOffset),
limit: Math.min(
Math.max(0, normalizedLimit),
Search.MAX_SEARCH_LIMIT
)
};
}
static normalizeIndexKey(key) {
const normalizedKey = String(key ?? '').trim();
if ( !Search.INDEX_KEY_REGEX.test(normalizedKey) ) {
throw new Error('Invalid search index key.');
}
return normalizedKey;
}
static normalizeIndexBaseUrl(url) {
const normalizedUrl = String(url ?? '').trim();
if ( normalizedUrl.length === 0 ) {
throw new Error('Invalid search index base URL.');
}
const parsedUrl = new URL(normalizedUrl, window.location.origin);
if ( !['http:', 'https:'].includes(parsedUrl.protocol) ) {
throw new Error('Invalid search index base URL.');
}
if ( parsedUrl.origin !== window.location.origin ) {
throw new Error(
'Search index base URL must use the current origin.');
}
return parsedUrl.href.replace(/\/$/, '');
}
static normalizeDocumentId(id) {
return Math.max(0, Math.floor(Number(id) || 0));
}
static async fetchIndexData(key, language = getPageLanguage()) {
const normalizedKey = Search.normalizeIndexKey(key);
const normalizedLanguage = normalizeLanguageKey(language);
const normalizedIndexBaseUrl = Search.normalizeIndexBaseUrl(
COLLECTION_INDEX_BASE_URL
);
const indexDataUrl =`${normalizedIndexBaseUrl}/` +
`${normalizedLanguage}/${normalizedKey}.json`;
const response = await fetch(indexDataUrl);
if ( !response.ok ) {
throw new Error(
`Failed to load search index '${normalizedKey}': ` +
`${response.status} ${response.statusText}`
);
}
const contentLength = Number(
response.headers.get('Content-Length') || 0);
if ( contentLength > Search.MAX_INDEX_FILE_SIZE_BYTES ) {
throw new Error(
`Search index '${normalizedKey}' exceeds the ` +
`maximum file size.`);
}
const data = await response.text();
const dataSize = new TextEncoder().encode(data).length;
if ( dataSize > Search.MAX_INDEX_FILE_SIZE_BYTES ) {
throw new Error(
`Search index '${normalizedKey}' exceeds the ` +
`maximum file size.`);
}
return { key: normalizedKey, data };
}
async loadIndex() {
this.loaded = false;
this.index = null;
this.collectionSize = 0;
try {
// Regenerate the index configuration.
const normalizedLanguage = getPageLanguage();
let indexConfig = structuredClone(INDEX_DOCUMENT_STORE_CONFIG);
indexConfig.language = normalizedLanguage in ISO_639_3166_LOOKUP ?
ISO_639_3166_LOOKUP[normalizedLanguage] : normalizedLanguage;
if ( CJK_ISO_3166.includes(indexConfig.language) ) {
indexConfig.encode = cjkTokenizer;
}
// Create an empty index using the same build-time index config.
this.index = new FlexSearch.Document(indexConfig);
const indexKeys = normalizedLanguage in LANGUAGE_INDEX_KEYS ?
LANGUAGE_INDEX_KEYS[normalizedLanguage] : [];
// Fetch all index files concurrently.
const indexData = await Promise.all(indexKeys.map(
function(key) {
return Search.fetchIndexData(key, normalizedLanguage);
}
));
// Import each index key using the fetched data.
for ( const { key, data } of indexData ) {
try {
await this.index.import(key, data ?? null);
} catch (error) {
throw new Error(
`Failed to import search index '${key}'.`,
{ cause: error });
}
}
// Set the collection size.
this.collectionSize = normalizedLanguage in COLLECTION_SIZES ?
COLLECTION_SIZES[normalizedLanguage] : 0;
this.loaded = true;
} catch (error) {
this.loaded = false;
this.index = null;
this.collectionSize = 0;
throw error;
}
}
getDocument(id) {
this.#assertIndexLoaded();
const normalizedId = Search.normalizeDocumentId(id);
if ( normalizedId >= this.collectionSize ) {
return null;
}
return this.index.get(normalizedId);
}
getDocuments(offset = 0, limit = 10) {
this.#assertIndexLoaded();
const {
offset: paginationOffset,
limit: paginationLimit
} = Search.normalizePagination(offset, limit);
let documents = [];
for (let i = paginationOffset; i < (
paginationOffset + paginationLimit); i++) {
const document = this.index.get(i);
if ( typeof document !== 'undefined' && document !== null ) {
documents.push(document);
}
}
return documents;
}
async getDocumentsByTags(tags, offset = 0, limit = 10, enrich = true) {
this.#assertIndexLoaded();
const normalizedTags = Search.normalizeTags(tags);
if ( normalizedTags.length === 0 ) {
return [];
}
const {
offset: paginationOffset,
limit: paginationLimit
} = Search.normalizePagination(offset, limit);
const normalizedEnrich = Search.normalizeEnrich(enrich);
return this.#deduplicateHits(await this.index.searchAsync({
tag: {
field: "tags",
tag: normalizedTags
},
offset: paginationOffset,
limit: paginationLimit,
enrich: normalizedEnrich
}), paginationLimit);
}
async query(searchQuery, options = {}) {
this.#assertIndexLoaded();
const {
enrich = true,
minSearchQueryLength = 2
} = options;
const {
offset: paginationOffset,
limit: paginationLimit
} = Search.normalizePagination(options.offset, options.limit);
const sanitizedSearchQuery = Search.sanitizeQuery(searchQuery);
const minQueryLength = Search.normalizeMinSearchQueryLength(
minSearchQueryLength
);
const normalizedEnrich = Search.normalizeEnrich(enrich);
if ( sanitizedSearchQuery.length < minQueryLength ) {
return [];
}
// Paginate unique documents after dedupe, not FlexSearch hit offsets.
const scanLimit = Math.min(
Math.max(this.collectionSize, 0),
Search.MAX_SEARCH_LIMIT
);
const allResults = this.#deduplicateHits(
await this.index.searchAsync(sanitizedSearchQuery, {
offset: 0,
limit: scanLimit,
enrich: normalizedEnrich
}),
scanLimit
);
return allResults.slice(
paginationOffset,
paginationOffset + paginationLimit
);
}
async queryAndFilterByTags(searchQuery, tags, options = {}) {
this.#assertIndexLoaded();
const {
enrich = true,
minSearchQueryLength = 2
} = options;
const {
offset: paginationOffset,
limit: paginationLimit
} = Search.normalizePagination(options.offset, options.limit);
const sanitizedSearchQuery = Search.sanitizeQuery(searchQuery);
const minQueryLength = Search.normalizeMinSearchQueryLength(
minSearchQueryLength
);
const normalizedTags = Search.normalizeTags(tags);
const normalizedEnrich = Search.normalizeEnrich(enrich);
if ( sanitizedSearchQuery.length < minQueryLength ||
normalizedTags.length === 0 ) {
return [];
}
// FlexSearch can return no hits for combined text + tag queries when
// the same term matches one document through a secondary field
// (e.g. content) while other documents match the term in primary
// fields. Intersect query results with tag-filtered documents instead.
const scanLimit = Math.min(
Math.max(this.collectionSize, 0),
Search.MAX_SEARCH_LIMIT
);
const queryResults = this.#deduplicateHits(
await this.index.searchAsync(sanitizedSearchQuery, {
offset: 0,
limit: scanLimit,
enrich: normalizedEnrich
}),
scanLimit
);
const tagResults = await this.getDocumentsByTags(
normalizedTags, 0, scanLimit, normalizedEnrich);
const tagIds = new Set(tagResults.map((doc) => doc.id));
const intersected = queryResults.filter((doc) => tagIds.has(doc.id));
return intersected.slice(
paginationOffset,
paginationOffset + paginationLimit
);
}
#deduplicateHits(hits, limit = Infinity) {
let docs = new Map();
let results = [];
if ( !Array.isArray(hits) ) {
return results;
}
for ( const hit of hits ) {
if ( !hit || !Array.isArray(hit.result) ) {
continue;
}
for ( const result of hit.result ) {
const doc = result && result.doc;
if ( !doc || typeof doc.id === 'undefined' ||
docs.has(doc.id) ) {
continue;
}
docs.set(doc.id, true);
results.push(doc);
if ( results.length >= limit ) {
return results;
}
}
}
return results;
}
}
The constants referenced in the search service class, for example COLLECTION_INDEX_BASE_URL, COLLECTION_SIZES, INDEX_DOCUMENT_STORE_CONFIG and so forth, are defined in $SITE_BASE/public/{env}/assets/js/site/config.js which is dynamically generated during the build and derived from the site configuration specified in the site.json file. Furthermore, the relevant <script> tag is automatically injected into the generated HTML files. For example, when the demo TravelBook static site is built, this config.js file will contain the following constant values.
const ASSETS_BASE_URL = "/assets";
const COLLECTION_INDEX_BASE_URL = "/assets/collection";
const COLLECTION_PAGINATION_SIZE = 10;
const COLLECTION_SIZES = {"en":15,"ja":15,"zh-tw":15};
const DEFAULT_LANGUAGE = "en";
const DOMAIN_NAME = "localhost:8080";
const INDEX_DOCUMENT_STORE_CONFIG = {"document":{"id":"id","tag":"tags","index":["name","description","content"],"store":true}};
const LANGUAGE_INDEX_KEYS = {"en":["1.doc","1.reg","1.tag","content.1.map","description.1.map","name.1.map"],"ja":["1.doc","1.reg","1.tag","content.1.map","description.1.map","name.1.map"],"zh-tw":["1.doc","1.reg","1.tag","content.1.map","description.1.map","name.1.map"]};
const MIN_SEARCH_QUERY_LENGTH = 2;
const SITE_VERSION = "0.0.1";
Finally, Teddy also provides a series of language-specific lookups and custom tokenizer functions to support, for example, Chinese, Japanese and Korean (CJK) languages. These language lookups and helper functions can be found in $TEDDY_BASE/system/assets/js/teddy/lang.js and, if site.html.inject.systemAssets is set to true in the site.json site configuration, is also automatically copied to the public distribution directory when your static site is built - specifically to $SITE_BASE/public/{env}/assets/js/vendors/teddy/{teddy.version}/lang.js - and the relevant <script> tag is automatically injected into the generated HTML files. These language lookups and helper functions are as follows.
// ISO 639 to ISO 3166 lookup.
const ISO_639_3166_LOOKUP = {
"en": "en",
"eng": "en",
"ja": "jp",
"jpn": "jp",
"ko": "kr",
"kor": "kr",
"zh": "cn",
"zh-hk": "cn",
"zh-tw": "cn"
}
// CJK ISO 3166 language codes.
const CJK_ISO_3166 = ['jp', 'kr', 'cn'];
// Unciode encodings for characters used by languages
// which rely on spaces to separate words.
const LANG_ALPHABETS = [
[0x30, 0x39], // 0-9
[0x41, 0x5a], // A-Z
[0x61, 0x7a], // a-z
[0xc0, 0x2af], // part of Latin-1 supplement / Latin extended A/B / IPA
[0x370, 0x52f] // Greek / Cyrillic / Cyrillic supplement
];
// Unicode encodings for characters used by languages
// where one character is one word.
const LANG_SINGLE_CHARS = [
[0xe00, 0x0e5b], // Thai
[0x3040, 0x309f], // Hiragana
[0x4e00, 0x9fff], // Chinese / Japanese / Korean
[0xac00, 0xd7af] // Hangul syllables
];
// Identify whether a given character is used by languages
// which rely on spaces to separate words.
function isAlphabet(n) {
for (let range of LANG_ALPHABETS) {
if (n >= range[0] && n <= range[1]) {
return true;
}
}
return false;
}
// Identify whether a given character is used by languages
// where one character is one word.
function isSingleChar(n) {
for (let range of LANG_SINGLE_CHARS) {
if (n >= range[0] && n <= range[1]) {
return true;
}
}
return false;
}
// Tokenizer that supports both types of languages.
function customTokenizer(str) {
const length = str.length;
const tokens = [];
let last = '';
for (let i = 0; i < length; i++) {
let code = str.charCodeAt(i);
if (isSingleChar(code)) {
if (last) {
if (last.length > 1) {
tokens.push(last.toLowerCase());
}
last = '';
}
tokens.push(str[i]);
} else if (isAlphabet(code)) {
last = last + str[i];
} else {
if (last) {
if (last.length > 1) {
tokens.push(last.toLowerCase());
}
last = '';
}
}
}
if (last) {
if (last.length > 1) {
tokens.push(last.toLowerCase());
}
last = '';
}
return tokens;
}
// Custom CJK tokenizer.
function cjkTokenizer(str) {
return str.replace(/[\x00-\x7F]/g, "").split("");
}
Instantiation
To create a new search service object, simply instantiate the search service class in your custom site-specific or theme-specific frontend JavaScript code as follows.
// Instantiate a new search service object.
const search = new Search();
Index Loading
After creating a new search service object, the next step is to load the correct localised FlexSearch document store index corresponding to the language of the active page. During the build, Teddy will write the FlexSearch index for each language to $SITE_BASE/public/{env}/assets/collection/{language}. The search service class provides a convenient method named loadIndex() that automatically detects the language of the active page and then loads the correct localised index, as follows.
// Load the correct localised index.
await search.loadIndex();
Document Retrieval
When indexing during the build, Teddy sorts the documents by the property key specified in site.collection.sort.key and in the order specified in site.collection.sort.order in the site.json site configuration. For example, the demo TravelBook site is configured to sort the collection of blog post pages in descending order of date last updated. This sort order is preserved when the localised index is loaded. To retrieve a range of documents with a given offset and limit, the search service class provides the getDocuments(offset: int, limit: int) method as follows.
getDocuments(offset = 0, limit = 10) {
this.#assertIndexLoaded();
const {
offset: paginationOffset,
limit: paginationLimit
} = Search.normalizePagination(offset, limit);
let documents = [];
for (let i = paginationOffset; i < (
paginationOffset + paginationLimit); i++) {
const document = this.index.get(i);
if ( typeof document !== 'undefined' && document !== null ) {
documents.push(document);
}
}
return documents;
}
For example, given offset = 0 and limit = 10, the search service will return the first 10 documents in the collection. In the case of the demo TravelBook site, it will return the 10 most recently updated documents (i.e. the latest 10 travel blog posts), in descending order of date, as follows.
// Retrieve the latest blog posts.
const latestDocs = await search.getDocuments(0, 10);
console.log(latestDocs);
// Console
[
{
"id": 0,
"name": "Serengeti National Park",
"description": "Tanzania is an East African country known for its vast wilderness areas. They include the plains of Serengeti National Park, a safari mecca populated by the 'big five' game (elephant, lion, leopard, buffalo, rhino), and Kilimanjaro National Park, home to Africa’s highest mountain. Offshore lie the tropical islands of Zanzibar, with Arabic influences, and Mafia, with a marine park home to whale sharks and coral reefs.",
"categoryLanguages": [
{
"id": "africa",
"name": "Africa"
}
],
"tags": [
"africa",
"serengeti",
"safari",
"Kilimanjaro",
"Zanzibar"
],
"date": "2025-03-04T12:00:00.000Z",
"displayDate": "4 March 2025",
"relUrl": "africa/tanzania",
"coverExists": true,
"cover": "assets/serengeti.jpg",
"author": "Teddy",
"authorUrl": "/about",
"content": "Lorem ipsum dolor sit amet consectetur..."
},
{
"id": 1,
"name": "Namib Desert",
"description": "Namibia, a country in southwest Africa, is distinguished by the Namib Desert along its Atlantic Ocean coast. The country is home to diverse wildlife, including a significant cheetah population. The capital, Windhoek, and coastal town Swakopmund contain German colonial-era buildings such as Windhoek's Christuskirche, built in 1907. In the north, Etosha National Park’s salt pan draws game including rhinos and giraffes.",
"categoryLanguages": [
{
"id": "africa",
"name": "Africa"
}
],
"tags": [
"africa",
"cheetah",
"Windhoek",
"Swakopmund",
"salt"
],
"date": "2025-03-03T12:00:00.000Z",
"displayDate": "3 March 2025",
"relUrl": "africa/namibia",
"coverExists": true,
"cover": "assets/desert.jpg",
"author": "Teddy",
"authorUrl": "/about",
"content": "Lorem ipsum dolor sit amet consectetur..."
},
{
"id": 2,
"name": "Mosi-oa-Tunya",
"description": "Zambia, in southern Africa, is a landlocked country of rugged terrain and diverse wildlife, with many parks and safari areas. On its border with Zimbabwe is famed Victoria Falls – indigenously called Mosi-oa-Tunya, or 'Smoke That Thunders' – plunging a misty 108m into narrow Batoka Gorge. Spanning the Zambezi River just below the falls is Victoria Falls Bridge, a spectacular viewpoint.",
"categoryLanguages": [
{
"id": "africa",
"name": "Africa"
}
],
"tags": [
"africa",
"Zambezi",
"Victoria",
"Mosi-oa-Tunya",
"Batoka"
],
"date": "2025-03-02T12:00:00.000Z",
"displayDate": "2 March 2025",
"relUrl": "africa/zambia",
"coverExists": true,
"cover": "assets/victoria-falls.jpg",
"author": "Teddy",
"authorUrl": "/about",
"content": "Lorem ipsum dolor sit amet consectetur..."
},
{
"id": 3,
"name": "Savannah Grasslands",
"description": "Kenya is a country in East Africa with coastline on the Indian Ocean. It encompasses savannah, lakelands, the dramatic Great Rift Valley and mountain highlands. It is also home to wildlife like lions, elephants and rhinos. From Nairobi, the capital, safaris visit the Maasai Mara Reserve, known for its annual wildebeest migrations, and Amboseli National Park, offering views of Tanzania's 5,895m Mt. Kilimanjaro.",
"categoryLanguages": [
{
"id": "africa",
"name": "Africa"
}
],
"tags": [
"africa",
"savannah",
"lakelands",
"lion",
"elephant"
],
"date": "2025-03-01T12:00:00.000Z",
"displayDate": "1 March 2025",
"relUrl": "africa/kenya",
"coverExists": true,
"cover": "assets/savannah.jpg",
"author": "Teddy",
"authorUrl": "/about",
"content": "Lorem ipsum dolor sit amet consectetur..."
},
{
"id": 4,
"name": "Kyoto Temples",
"description": "Japan is an island country in East Asia. Located in the Pacific Ocean off the northeast coast of the Asian mainland, it is bordered on the west by the Sea of Japan and extends from the Sea of Okhotsk in the north to the East China Sea in the south.",
"categoryLanguages": [
{
"id": "asia",
"name": "Asia"
}
],
"tags": [
"asia",
"sushi",
"samurai",
"hokkaido",
"kyushu",
"honshu",
"anime",
"kyoto",
"temple",
"fuji",
"onsen",
"kanji"
],
"date": "2025-02-04T12:00:00.000Z",
"displayDate": "4 February 2025",
"relUrl": "asia/japan",
"coverExists": true,
"cover": "assets/kyoto.jpg",
"author": "Teddy",
"authorUrl": "/about",
"content": "Lorem ipsum dolor sit amet consectetur..."
},
{
"id": 5,
"name": "Bustling Hong Kong",
"description": "Hong Kong is a special administrative region of China. With 7.4 million residents of various nationalities in a 1,104-square-kilometre territory, Hong Kong is the fourth most densely populated region in the world.",
"categoryLanguages": [
{
"id": "asia",
"name": "Asia"
}
],
"tags": [
"asia",
"china",
"sar"
],
"date": "2025-02-03T12:00:00.000Z",
"displayDate": "3 February 2025",
"relUrl": "asia/hong-kong",
"coverExists": true,
"cover": "assets/hk.jpg",
"author": "Teddy",
"authorUrl": "/about",
"content": "Lorem ipsum dolor sit amet consectetur..."
},
{
"id": 6,
"name": "Kuala Lumpur",
"description": "Malaysia is a Southeast Asian country occupying parts of the Malay Peninsula and the island of Borneo. It's known for its beaches, rainforests and mix of Malay, Chinese, Indian and European cultural influences. The capital, Kuala Lumpur, is home to colonial buildings, busy shopping districts such as Bukit Bintang and skyscrapers such as the iconic, 451m-tall Petronas Twin Towers.",
"categoryLanguages": [
{
"id": "asia",
"name": "Asia"
}
],
"tags": [
"asia",
"petronas",
"malay",
"borneo"
],
"date": "2025-02-02T12:00:00.000Z",
"displayDate": "2 February 2025",
"relUrl": "asia/malaysia",
"coverExists": true,
"cover": "assets/kuala-lumpur.jpg",
"author": "Teddy",
"authorUrl": "/about",
"content": "Lorem ipsum dolor sit amet consectetur..."
},
{
"id": 7,
"name": "Sizzling Singapore",
"description": "Singapore, officially the Republic of Singapore, is an island country and city-state in Southeast Asia. The country's territory comprises one main island, 63 satellite islands and islets, and one outlying islet.",
"categoryLanguages": [
{
"id": "asia",
"name": "Asia"
}
],
"tags": [
"asia",
"strait",
"republic",
"island"
],
"date": "2025-02-01T12:00:00.000Z",
"displayDate": "1 February 2025",
"relUrl": "asia/singapore",
"coverExists": true,
"cover": "assets/singapore.jpg",
"author": "Teddy",
"authorUrl": "/about",
"content": "Lorem ipsum dolor sit amet consectetur..."
},
{
"id": 8,
"name": "Hobbiton",
"description": "New Zealand is an island country in the southwestern Pacific Ocean. It consists of two main landmasses—the North Island and the South Island —and over 700 smaller islands.",
"categoryLanguages": [
{
"id": "australia",
"name": "Australia"
}
],
"tags": [
"australia",
"lotr",
"hobbits",
"wellington",
"auckland"
],
"date": "2025-01-01T12:00:00.000Z",
"displayDate": "1 January 2025",
"relUrl": "australia/new-zealand",
"coverExists": true,
"cover": "assets/hobbiton.jpg",
"author": "Teddy",
"authorUrl": "/about",
"content": "Lorem ipsum dolor sit amet consectetur..."
},
{
"id": 9,
"name": "Northern Lights",
"description": "Finland is a Northern European nation bordering Sweden, Norway and Russia. Its capital, Helsinki, occupies a peninsula and surrounding islands in the Baltic Sea. Helsinki is home to the 18th-century sea fortress Suomenlinna, the fashionable Design District and diverse museums. The Northern Lights can be seen from the country's Arctic Lapland province, a vast wilderness with national parks and ski resorts.",
"categoryLanguages": [
{
"id": "europe",
"name": "Europe"
}
],
"tags": [
"europe",
"helsinki",
"baltic",
"Suomenlinna",
"Lapland"
],
"date": "2024-12-02T12:00:00.000Z",
"displayDate": "2 December 2024",
"relUrl": "europe/finland",
"coverExists": true,
"cover": "assets/northern-lights.jpg",
"author": "Teddy",
"authorUrl": "/about",
"content": "Lorem ipsum dolor sit amet consectetur..."
}
]
Document Schema
The schema of a document object that is returned by the search service, as demonstrated above, is as follows.
{
"id": int,
"name": string,
"description": string,
"categoryLanguages": object{id: string, name: string},
"tags": list[string],
"date": string,
"displayDate": string,
"relUrl": string,
"coverExists": boolean
"cover": string,
"author": string,
"authorUrl": string,
"content": string
}
For example:
{
"id": 2,
"name": "Mosi-oa-Tunya",
"description": "Zambia, in southern Africa, is a landlocked country of rugged terrain and diverse wildlife, with many parks and safari areas. On its border with Zimbabwe is famed Victoria Falls – indigenously called Mosi-oa-Tunya, or 'Smoke That Thunders' – plunging a misty 108m into narrow Batoka Gorge. Spanning the Zambezi River just below the falls is Victoria Falls Bridge, a spectacular viewpoint.",
"categoryLanguages": [
{
"id": "africa",
"name": "Africa"
}
],
"tags": [
"africa",
"Zambezi",
"Victoria",
"Mosi-oa-Tunya",
"Batoka"
],
"date": "2025-03-02T12:00:00.000Z",
"displayDate": "2 March 2025",
"relUrl": "africa/zambia",
"coverExists": true,
"cover": "assets/victoria-falls.jpg",
"author": "Teddy",
"authorUrl": "/about",
"content": "Lorem ipsum dolor sit amet consectetur..."
}
Note that the content value is not the raw content of the page markdown-based file, but rather a processed version of it with all metadata, HTML tags and common punctuation characters removed for the purposes of indexing. Thus content should not be used to display the content of your page to users. To display the content of a page, you should use the ${page.content} placeholder in your HTML-based theme templates as described in the pages guide. Also note that the content key is only returned in the document object if site.collection.index.content is set to true in the site.json site configuration.
Extending the Document Schema
To extend the document schema with arbitrary keys from the frontmatter of your markdown-based pages, simply add the name of the custom key to the site.collection.index.documentStore.document.index list defined in the site.json site configuration, as shown in the following example.
{
"site": {
...
"collection": {
"enabled": true,
"pagesDir": ".",
"index": {
"content": true,
"documentStore": {
"document": {
"id": "id",
"tag": "tags",
"index": [
"name",
"description",
"eventDate",
"bookingUrl",
"sponsorUrl"
],
"store": true
}
}
},
...
}
...
}
}
Search Query
To perform a free-text search across the configured indexed fields (for example name, description and content) of the documents with a given offset and limit, the search service class provides the query(searchQuery, options = {}) method as follows.
async query(searchQuery, options = {}) {
this.#assertIndexLoaded();
const {
enrich = true,
minSearchQueryLength = 2
} = options;
const {
offset: paginationOffset,
limit: paginationLimit
} = Search.normalizePagination(options.offset, options.limit);
const sanitizedSearchQuery = Search.sanitizeQuery(searchQuery);
const minQueryLength = Search.normalizeMinSearchQueryLength(
minSearchQueryLength
);
const normalizedEnrich = Search.normalizeEnrich(enrich);
if ( sanitizedSearchQuery.length < minQueryLength ) {
return [];
}
// Paginate unique documents after dedupe, not FlexSearch hit offsets.
const scanLimit = Math.min(
Math.max(this.collectionSize, 0),
Search.MAX_SEARCH_LIMIT
);
const allResults = this.#deduplicateHits(
await this.index.searchAsync(sanitizedSearchQuery, {
offset: 0,
limit: scanLimit,
enrich: normalizedEnrich
}),
scanLimit
);
return allResults.slice(
paginationOffset,
paginationOffset + paginationLimit
);
}
The enrich parameter, when set to false, will return only the IDs of those documents that fulfil the search query request. When set to true, the full document object of those documents that fulfil the search query request will be returned in the response. For example, given searchQuery = 'snow', offset = 0, limit = 10 and enrich = true, the search service will return the following documents.
// Search for blog posts containing the word 'snow'.
const docs = await search.query(searchQuery, {
offset: 0,
limit: 10,
minSearchQueryLength: MIN_SEARCH_QUERY_LENGTH
});
console.log(docs);
// Console
[
{
"id": 13,
"name": "Snow-capped Santiago",
"description": "Chile is a long, narrow country stretching along South America's western edge, with more than 6,000km of Pacific Ocean coastline. Santiago, its capital, sits in a valley surrounded by the Andes and Chilean Coast Range mountains. The city's palm-lined Plaza de Armas contains the neoclassical cathedral and the National History Museum. The massive Parque Metropolitano offers swimming pools, a botanical garden and zoo.",
"categoryLanguages": [
{
"id": "south-america",
"name": "South America"
}
],
"tags": [
"south-america",
"chile",
"pacific",
"Armas"
],
"date": "2024-10-02T12:00:00.000Z",
"displayDate": "2 October 2024",
"relUrl": "south-america/chile",
"coverExists": true,
"cover": "assets/santiago.jpg",
"author": "Teddy",
"authorUrl": "/about",
"content": "Lorem ipsum dolor sit amet consectetur..."
},
{
"id": 12,
"name": "Uyuni Salt Flats",
"description": "Bolivia is a country in central South America, with a varied terrain spanning Andes Mountains, the Atacama Desert and Amazon Basin rainforest. At more than 3,500m, its administrative capital, La Paz, sits on the Andes' Altiplano plateau with snow-capped Mt. Illimani in the background. Nearby is glass-smooth Lake Titicaca, the continent's largest lake, straddling the border with Peru.",
"categoryLanguages": [
{
"id": "south-america",
"name": "South America"
}
],
"tags": [
"south-america",
"andes",
"Atacama",
"Altiplano",
"Illimani",
"Titicaca",
"Uyuni",
"salt"
],
"date": "2024-10-03T12:00:00.000Z",
"displayDate": "3 October 2024",
"relUrl": "south-america/bolivia",
"coverExists": true,
"cover": "assets/uyuni-salt-flats.jpg",
"author": "Teddy",
"authorUrl": "/about",
"content": "Lorem ipsum dolor sit amet consectetur..."
}
]
Note that the query() method will first sanitize the search query by removing a selection of illegal characters including, but not limited to, currency symbols, brackets, slashes, and other common punctuation characters.
Filtering
To perform tag-based filtering of the documents, including filtering on category keys, the search service class provides the getDocumentsByTags(tags: list[string], offset: int, limit: int, enrich: boolean) method, where the tag parameter is a list of tag and/or category key strings, as follows.
async getDocumentsByTags(tags, offset = 0, limit = 10, enrich = true) {
this.#assertIndexLoaded();
const normalizedTags = Search.normalizeTags(tags);
if ( normalizedTags.length === 0 ) {
return [];
}
const {
offset: paginationOffset,
limit: paginationLimit
} = Search.normalizePagination(offset, limit);
const normalizedEnrich = Search.normalizeEnrich(enrich);
return this.#deduplicateHits(await this.index.searchAsync({
tag: {
field: "tags",
tag: normalizedTags
},
offset: paginationOffset,
limit: paginationLimit,
enrich: normalizedEnrich
}), paginationLimit);
}
For example, given tags = ['asia', 'europe'], offset = 0, limit = 10 and enrich = true, the search service will return all documents in the categories asia or europe as follows.
// Filter documents based on the categories 'asia' or 'europe'.
const docs = await search.getDocumentsByTags(['asia', 'europe'], 0, 10, true);
console.log(docs);
// Console
[
{
"id": 4,
"name": "Kyoto Temples",
"description": "Japan is an island country in East Asia. Located in the Pacific Ocean off the northeast coast of the Asian mainland, it is bordered on the west by the Sea of Japan and extends from the Sea of Okhotsk in the north to the East China Sea in the south.",
"categoryLanguages": [
{
"id": "asia",
"name": "Asia"
}
],
"tags": [
"asia",
"sushi",
"samurai",
"hokkaido",
"kyushu",
"honshu",
"anime",
"kyoto",
"temple",
"fuji",
"onsen",
"kanji"
],
"date": "2025-02-04T12:00:00.000Z",
"displayDate": "4 February 2025",
"relUrl": "asia/japan",
"coverExists": true,
"cover": "assets/kyoto.jpg",
"author": "Teddy",
"authorUrl": "/about",
"content": "Lorem ipsum dolor sit amet consectetur..."
},
{
"id": 5,
"name": "Bustling Hong Kong",
"description": "Hong Kong is a special administrative region of China. With 7.4 million residents of various nationalities in a 1,104-square-kilometre territory, Hong Kong is the fourth most densely populated region in the world.",
"categoryLanguages": [
{
"id": "asia",
"name": "Asia"
}
],
"tags": [
"asia",
"china",
"sar"
],
"date": "2025-02-03T12:00:00.000Z",
"displayDate": "3 February 2025",
"relUrl": "asia/hong-kong",
"coverExists": true,
"cover": "assets/hk.jpg",
"author": "Teddy",
"authorUrl": "/about",
"content": "Lorem ipsum dolor sit amet consectetur..."
},
{
"id": 6,
"name": "Kuala Lumpur",
"description": "Malaysia is a Southeast Asian country occupying parts of the Malay Peninsula and the island of Borneo. It's known for its beaches, rainforests and mix of Malay, Chinese, Indian and European cultural influences. The capital, Kuala Lumpur, is home to colonial buildings, busy shopping districts such as Bukit Bintang and skyscrapers such as the iconic, 451m-tall Petronas Twin Towers.",
"categoryLanguages": [
{
"id": "asia",
"name": "Asia"
}
],
"tags": [
"asia",
"petronas",
"malay",
"borneo"
],
"date": "2025-02-02T12:00:00.000Z",
"displayDate": "2 February 2025",
"relUrl": "asia/malaysia",
"coverExists": true,
"cover": "assets/kuala-lumpur.jpg",
"author": "Teddy",
"authorUrl": "/about",
"content": "Lorem ipsum dolor sit amet consectetur..."
},
{
"id": 7,
"name": "Sizzling Singapore",
"description": "Singapore, officially the Republic of Singapore, is an island country and city-state in Southeast Asia. The country's territory comprises one main island, 63 satellite islands and islets, and one outlying islet.",
"categoryLanguages": [
{
"id": "asia",
"name": "Asia"
}
],
"tags": [
"asia",
"strait",
"republic",
"island"
],
"date": "2025-02-01T12:00:00.000Z",
"displayDate": "1 February 2025",
"relUrl": "asia/singapore",
"coverExists": true,
"cover": "assets/singapore.jpg",
"author": "Teddy",
"authorUrl": "/about",
"content": "Lorem ipsum dolor sit amet consectetur..."
},
{
"id": 9,
"name": "Northern Lights",
"description": "Finland is a Northern European nation bordering Sweden, Norway and Russia. Its capital, Helsinki, occupies a peninsula and surrounding islands in the Baltic Sea. Helsinki is home to the 18th-century sea fortress Suomenlinna, the fashionable Design District and diverse museums. The Northern Lights can be seen from the country's Arctic Lapland province, a vast wilderness with national parks and ski resorts.",
"categoryLanguages": [
{
"id": "europe",
"name": "Europe"
}
],
"tags": [
"europe",
"helsinki",
"baltic",
"Suomenlinna",
"Lapland"
],
"date": "2024-12-02T12:00:00.000Z",
"displayDate": "2 December 2024",
"relUrl": "europe/finland",
"coverExists": true,
"cover": "assets/northern-lights.jpg",
"author": "Teddy",
"authorUrl": "/about",
"content": "Lorem ipsum dolor sit amet consectetur..."
},
{
"id": 10,
"name": "Sognefjord and Viking Ships",
"description": "Norway is a Scandinavian country encompassing mountains, glaciers and deep coastal fjords. Oslo, the capital, is a city of green spaces and museums. Preserved 9th-century Viking ships are displayed at Oslo’s Viking Ship Museum. Bergen, with colorful wooden houses, is the starting point for cruises to the dramatic Sognefjord. Norway is also known for fishing, hiking and skiing, notably at Lillehammer’s Olympic resort.",
"categoryLanguages": [
{
"id": "europe",
"name": "Europe"
}
],
"tags": [
"europe",
"scandinavia",
"fjord",
"viking",
"Sognefjord"
],
"date": "2024-12-01T12:00:00.000Z",
"displayDate": "1 December 2024",
"relUrl": "europe/norway",
"coverExists": true,
"cover": "assets/viking-ship.jpg",
"author": "Teddy",
"authorUrl": "/about",
"content": "Lorem ipsum dolor sit amet consectetur..."
}
]
Note that boolean OR logic is applied when multiple tags are provided in the tags list.
Search Query and Filter
To combine free-text search and tag-based filtering in one query, the search service class provides the queryAndFilterByTags(searchQuery, tags, options = {}) method. For example, given searchQuery = 'island', tags = ['australia', 'europe'], offset = 0, limit = 10 and enrich = true, the search service will return all documents that contain the phrase island AND belong to either the categories australia OR europe as follows.
// Search for blog posts containing the phrase 'island' and filter based on the categories 'australia' or 'europe'.
const docs = await search.queryAndFilterByTags('island', ['australia', 'europe'], {
offset: 0,
limit: 10,
minSearchQueryLength: MIN_SEARCH_QUERY_LENGTH
});
console.log(docs);
// Console
[
{
"id": 8,
"name": "Hobbiton",
"description": "New Zealand is an island country in the southwestern Pacific Ocean. It consists of two main landmasses—the North Island and the South Island —and over 700 smaller islands.",
"categoryLanguages": [
{
"id": "australia",
"name": "Australia"
}
],
"tags": [
"australia",
"lotr",
"hobbits",
"wellington",
"auckland"
],
"date": "2025-01-01T12:00:00.000Z",
"displayDate": "1 January 2025",
"relUrl": "australia/new-zealand",
"coverExists": true,
"cover": "assets/hobbiton.jpg",
"author": "Teddy",
"authorUrl": "/about",
"content": "Lorem ipsum dolor sit amet consectetur..."
}
]
FlexSearch API
For further information on the FlexSearch search API, and to view the full range of search features available, please visit https://github.com/nextapps-de/flexsearch.