#!/usr/bin/env python3 """Text-index design benchmark, standard library only; uses generated synthetic docs.""" import csv, gc, json, math, random, re, statistics, sys, time, tracemalloc from array import array from collections import Counter from datetime import datetime, timezone from pathlib import Path HERE = Path(__file__).resolve().parent SIZES = [2000, 8000, 20000] REPEATS = 3 QUERY_REPEATS = 10 TOP_K = 10 TOKEN_RE = re.compile(r"[a-z0-9_]+") TOPICS = { "privacy": ["encryption", "keyrotation", "threatmodel"], "network": ["routing", "packetloss", "throughputcpu"], "retrieval": ["indexing", "invertedindex", "relevance"], "reliability": ["failover", "rollback", "resilience"], "compute": ["scheduler", "parallelism", "workqueue"], "storage": ["snapshot", "checksum", "retention"], "security": ["authentication", "authorization", "auditlog"], "learning": ["hypothesis", "validation", "reproducibility"], } TOPIC_NAMES = list(TOPICS) COMMON = ("system service process request response resource latency throughput failure recovery " "memory disk cache network model document query result quality measure test control dataset " "method evidence update version configuration worker queue operation event metric monitor policy " "storage runtime interface pipeline analysis compute metadata performance scaling report experiment " "deployment privacy security index search token ranking knowledge retrieval checksum validation " "servicelevel access control planning automation resilience policy decision").split() def tokenize(s): return TOKEN_RE.findall(s.lower()) def corpus(n): docs, labels = [], [] for i in range(n): topic = TOPIC_NAMES[i % len(TOPIC_NAMES)] rng = random.Random(20261028 + i) noise = [rng.choice(COMMON) for _ in range(56)] docs.append("document %d topic %s " % (i, topic) + " ".join(TOPICS[topic] + noise)) labels.append(topic) return docs, labels def build_set(docs): post, tfs = {}, [] for i, doc in enumerate(docs): tf = dict(Counter(tokenize(doc))) tfs.append(tf) for term in tf: post.setdefault(term, set()).add(i) return {"kind":"set", "post":post, "tfs":tfs, "n":len(docs)} def build_array(docs): post = {} for i, doc in enumerate(docs): for term, tf in Counter(tokenize(doc)).items(): if term not in post: post[term] = (array("I"), array("I")) post[term][0].append(i) post[term][1].append(tf) return {"kind":"array", "post":post, "n":len(docs)} def get_idf(ix, term): v = ix["post"].get(term) if v is None: return 0.0 df = len(v) if ix["kind"] == "set" else len(v[0]) return math.log((ix["n"] + 1) / (df + 1)) + 1.0 def search_set(ix, q, k=TOP_K): post = ix["post"] if not q or any(t not in post for t in q): return [] groups = sorted((post[t] for t in q), key=len) cand = set(groups[0]) for g in groups[1:]: cand.intersection_update(g) if not cand: return [] weights = [(t, get_idf(ix,t)) for t in q] out = [] for i in cand: tf = ix["tfs"][i] out.append((i, sum(tf.get(t,0) * w for t,w in weights))) return sorted(out, key=lambda x:(-x[1], x[0]))[:k] def search_array(ix, q, k=TOP_K): post = ix["post"] if not q or any(t not in post for t in q): return [] ids, tfs = post[q[0]] w = get_idf(ix,q[0]) cur_i, cur_s = list(ids), [tf*w for tf in tfs] for term in q[1:]: other_i, other_tf = post[term] w = get_idf(ix,term) a=b=0; ni=[]; ns=[] while a < len(cur_i) and b < len(other_i): x,y = cur_i[a], other_i[b] if x == y: ni.append(x); ns.append(cur_s[a] + other_tf[b]*w); a+=1; b+=1 elif x < y: a+=1 else: b+=1 cur_i,cur_s = ni,ns if not cur_i: return [] return sorted(zip(cur_i,cur_s), key=lambda x:(-x[1],x[0]))[:k] BUILDERS = {"set":build_set, "array":build_array} SEARCHERS = {"set":search_set, "array":search_array} QUERIES = [(t, TOPICS[t][:2]) for t in TOPIC_NAMES] def main(): started = datetime.now(timezone.utc).isoformat() rows, validation = [], {} for n in SIZES: print("Generating %d synthetic documents" % n, flush=True) docs, labels = corpus(n) rankings = {} for kind in ("set","array"): times, peaks = [], [] for rep in range(REPEATS): gc.collect() tracemalloc.start() t0 = time.perf_counter() ix = BUILDERS[kind](docs) times.append(time.perf_counter()-t0) _, peak = tracemalloc.get_traced_memory() tracemalloc.stop() peaks.append(peak / 1048576) del ix gc.collect() print("n=%d kind=%s build=%d/%d seconds=%.4f peak_traced_mb=%.2f" % (n,kind,rep+1,REPEATS,times[-1],peaks[-1]), flush=True) # Build a clean index for query testing; excluded from build-time samples above. ix = BUILDERS[kind](docs) searcher = SEARCHERS[kind] qtimes, hits, tops = [], [], {} for _ in range(QUERY_REPEATS): for topic,q in QUERIES: t0 = time.perf_counter() result = searcher(ix,q) qtimes.append((time.perf_counter()-t0)*1000) ids = [docid for docid,_ in result] tops[topic] = ids relevant = sum(labels[docid] == topic for docid in ids) hits.append(relevant/len(ids) if ids else 0.0) if len(ids) != TOP_K or relevant != len(ids): raise RuntimeError("Top-10 relevance validation failed: n=%d kind=%s topic=%s" % (n,kind,topic)) rankings[kind] = tops qs = sorted(qtimes); p95 = qs[min(len(qs)-1,int(0.95*len(qs)))] rows.append({ "documents":n,"index_type":kind,"build_repeats":REPEATS, "median_build_seconds":statistics.median(times), "mean_build_seconds":statistics.mean(times), "build_stdev_seconds":statistics.pstdev(times), "median_peak_traced_mb":statistics.median(peaks), "docs_indexed_per_second":n/statistics.median(times), "median_query_ms":statistics.median(qtimes),"p95_query_ms":p95, "hit_at_10":statistics.mean(hits),"unique_terms":len(ix["post"]), "posting_entries":sum(len(v) if kind=="set" else len(v[0]) for v in ix["post"].values()) }) print("RESULT n=%d kind=%s build=%.4fs query_median=%.4fms query_p95=%.4fms hit@10=%.3f" % (n,kind,statistics.median(times),statistics.median(qtimes),p95,statistics.mean(hits)),flush=True) del ix for topic,_ in QUERIES: if rankings["set"][topic] != rankings["array"][topic]: raise RuntimeError("Index designs rank results differently: n=%d topic=%s" % (n,topic)) validation[str(n)] = {"rankings_identical":True,"queries_checked":len(QUERIES), "all_top10_relevant":True} del docs,labels gc.collect() finished = datetime.now(timezone.utc).isoformat() obj = { "experiment":"text-indexing-retrieval-benchmark","started_utc":started,"finished_utc":finished, "host":"darkoza001","python":sys.version, "dataset":{"type":"deterministically generated synthetic technical documents","user_data_used":False, "sizes":SIZES,"topics":len(TOPICS),"approx_tokens_per_document":60, "query_type":"two topic-specific signature terms with known synthetic labels"}, "methods":{"set":"Python set posting lists plus per-document term-frequency dictionaries; set intersection", "array":"sorted document IDs and frequencies in compact unsigned-int arrays; two-pointer intersection", "build_repeats":REPEATS,"query_repeats_per_topic":QUERY_REPEATS,"top_k":TOP_K}, "results":rows,"validation":validation, "limitations":["Synthetic corpus and known-label queries test mechanics, not real-world search relevance.", "tracemalloc measures Python-traced allocations, not total resident memory.", "Only two index designs, three sizes and one host were tested.", "No scientific novelty or LLM reasoning claim follows from this benchmark."]} (HERE/"results.json").write_text(json.dumps(obj,indent=2)+"\n") with (HERE/"results.csv").open("w",newline="") as f: fields=["documents","index_type","median_build_seconds","mean_build_seconds","build_stdev_seconds", "median_peak_traced_mb","docs_indexed_per_second","median_query_ms","p95_query_ms", "hit_at_10","unique_terms","posting_entries"] w=csv.DictWriter(f,fieldnames=fields); w.writeheader() for r in rows: w.writerow({x:r[x] for x in fields}) lines=["# Text indexing and retrieval benchmark — measured results","", "- Host: darkoza001; Python "+sys.version.split()[0], "- Window UTC: "+started+" to "+finished, "- Data: synthetic corpus only; no user files read; no dependencies installed.", "- Per size: two index designs, %d build repetitions each, %d queries repeated %d times."%(REPEATS,len(QUERIES),QUERY_REPEATS), "- Validation: top-10 results were relevant for every controlled query; both designs had identical rankings.","", "## Measurements","","| Documents | Index | Median build (s) | Docs/s | Peak traced MB | Query median (ms) | Query P95 (ms) | Hit@10 |", "|---:|---|---:|---:|---:|---:|---:|---:|"] for r in rows: lines.append("| %s | %s | %.4f | %s | %.2f | %.4f | %.4f | %.3f |"% (format(r["documents"],","),r["index_type"],r["median_build_seconds"], format(round(r["docs_indexed_per_second"]),","),r["median_peak_traced_mb"], r["median_query_ms"],r["p95_query_ms"],r["hit_at_10"])) lines += ["","## Decision","", "- Compare build throughput, memory and query latency together; do not select an index based on one metric alone.", "- Keep the representation configurable until a repeat run confirms the best choice.", "- Before production use, validate against permission-safe representative documents and human-judged relevance queries.", "","## Limits","", "- The corpus is synthetic and the queries are deliberately easy; Hit@10=1.0 checks implementation correctness, not real-user retrieval quality.", "- Python-traced peak memory does not equal process RSS.", "- No service or scheduled job was installed."] (HERE/"report.md").write_text("\n".join(lines)+"\n") print("FINAL_RESULT "+json.dumps({"report":str(HERE/"report.md"), "results_json":str(HERE/"results.json"),"results_csv":str(HERE/"results.csv"), "results":rows,"validation":validation}),flush=True) if __name__=="__main__": main()