Memelang: トークン削減型のクエリ言語

Memelang Memelangは、LLMが毎日数十億ものSQLクエリを生成する際のコンピューティングコストを大幅に削減する、トークン削減型のクエリ言語です。低トークン数のMemelangは中間表現として出力され、その後プログラムによってデータベース用の高トークン数SQLに変換されます。例: Memelang (20トークン) roles actor :$a="Mark Hamill";movie _;@ @ @;actor !$a SQL (36トークン) SELECT x.actor,x.movie,y.movie,y.actor FROM roles AS x,roles AS y WHERE x.actor='Mark Hamill' AND y.id!=x.id AND y.movie=x.movie ビデオ概要 arXiv論文 GitHubリポジトリ 特許仕様書 すべてのコードをコピー # Memelangがどのように機能するかをLLMに尋ねるために、このコードをコピー＆ペーストしてください # info@memelang.net | (c)2026 HOLTWORK LLC | 特許取得済み # MEMELANGはLLMのテキストtoSQLのためのトークン削減型DSL IRです # Axial grammar: Axis2 -> Axis1 -> Axis0 -> Cell # 空白は構文的であり、「新しいCell」をトリガーします # オペレーター/コンパレーター/カンマ/フラグと値の間に空白を入れないでください MEMELANG_VER = 11.04 basic_syntax = '[table WS] [column WS] [":$" var][":" ("min"|"max"|"cnt"|"sum"|"avg"|"last"|"grp")] [":" ("asc"|"des")] ["<=>" "" string """] [("="|"!="|">"|"<"|">="|"<="|"~"|"!~") (string|int|float|("$" var)|"@"|"_")] ";"' examples = ''' %mode=tab; roles id :int>0; rating :DESC="Decimal 0-5 star rating of performance";:dec>0.0;<=5; actor :DESC="Actor's full name";:str; movie :DESC="Movie's full name";:str; character :DESC="Character's full name";:str;; actors id :int>0; name :DESC="Actor's full name";:str; age :DESC="Actor's age in years";:int>=0;<200;; movies id :int>0; description :DESC="Brief description of movie plot";:str; year :DESC="Year of production AD";:int>1800;<2100; genre scifi,drama,comedy,documentary;:str; title :DESC="Full movie title";:str;; actors name _; roles actor @;; movies title _; roles movie @;; roles id :gct=1;; roles movie :grp; actor :grp; character :gct=1;; actors id :gct=1;; movies id :gct=1;; %mode=qry; """ すべての映画 """ movies _ _;; """ すべての役 """ roles _ _;; """ 映画のタイトルと説明 """ movies title _; description _;; """ 俳優の名前と年齢 """ actors name _; age _;; """ 41歳以上の俳優 """ actors age >=41; _;; """ 役 567 と 8901 """ roles id 567,8901; _;; """ 映画の説明にディストピア社会の物語が含まれるもの sim>.33 """ movies description <=>'dystopian'<0.33; _;; """ 1977年または1980年に公開された「Star」というタイトルの映画 """ movies title ~"Star"; year 1977,1980; _;; """ Anaという名前で20歳から35歳までの俳優 """ actors name ~"Ana"; age >=20;<=35; _;; """ 1980年より前の映画で評価が1.5未満の役 """ movies year <1980; title _; roles movie @; rating <1.5; _;; """ 役の評価を降順、映画を降順にソート """ roles rating :des; movie :des;; """ 1970年より前のすべての映画を昇順で並べ替え """ movies year :asc<1970; _;; """ 平均パフォーマー評価が4.2以上 """ roles rating :avg>=4.2; actor :grp;; """ 俳優ごとの最小役評価、低から高へ """ roles rating :min:asc; actor :grp;; """ ロボットが登場する映画で評価が3以上の役 """ movies description <=>'robot'<=$sim; title _; roles movie @; rating >=3;; """ ブルース・ウィリスまたはユマ・サーマンと共演した役 """ roles actor :$a~"Bruce Willis","Uma Thurman"; movie _;@ @ @; actor !$a;; """ 1980年より前の戦争物語：最小役評価によるトップ12映画 """ movies year <1980; description <=>'war'<=$sim; title :grp; roles movie @; rating :min:des;%beg=0;%lim=12;; """ 映画「Hero」または「House of Flying Daggers」で、俳優名にLiが含まれる役、俳優名A-Z """ movies title "Hero","House of Flying Daggers"; roles movie @; actor :asc~"Li";; """ 1900年から2000年の間にロボットについて言及するタイトルの映画 """ movies title ~"Hero"; description <=>'robot'; year >=1900; <=2000;; %tab=movies; #%val; %col=title; ~"Hero"; %col=description; <=>'robot'; %col=year; >=1900; <=2000;; %tab=movies; #title #description #year; ~"Hero" <=>'robot' >=1900; <=2000;; %tab=movies; #title; ~"Hero"; #description; <=>'robot'; #year; >=1900; <=2000;; #%tab #%val; movies :#title~"Hero"; :#description<=>'robot'; :#year>=1900; <=2000;; ''' import re, sys, json from typing import Optional, Union, List, Iterator, Pattern, Any Err = SyntaxError ### SYNTAX ### CELL_PATTERN = ( ('QUO', r'"(?:[^"\ ]|\.)*"'), ('EMB', r'\['(?:-?\d+(?:\.\d+)?)(?:\s*,\s*-?\d+(?:\.\d+)?)*\]'), ('MOD', r'<->|<=>|<#>'), ('CMP', r'>=|<=|!~|!=|=|>|<|~|!'), ('BIND', r':\$\w+'), ('FLAG', r':[a-zA-Z]+'), ('VAR', r'\$\w+'), ('REL', r'@\d?|\^'), ('WLD', r'_'), ('EVAR', r'%[a-zA-Z0-9_]+'), ('SLOT', r'#%?[a-zA-Z0-9_]+'), ('ASSN', r':#[a-zA-Z0-9_]+'), ('TIM', r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}'), ('DEC', r'-?\d*\.\d+'), ('INT', r'-?\d+'), ('ALN', r'[A-Za-z][A-Za-z0-9_]*'), ('OR', r','), ('WS', r'\s+'), ('MISMATCH', r'.'), ) CANON = {'!':'!='} CELL_REGEX=re.compile("|".join(f"(?P<{k}>{p})" for k, p in CELL_PATTERN)) PAD_MODES = {'qry','tab'} FLAG_KINDS = {'FLAG','BIND','EVAR','ASSN'} LIT_KINDS = {'TIM','DEC','INT','ALN','QUO','EMB'} VAR_KINDS = {'VAR','WLD','REL','EVAR','SLOT'} DAT_KINDS = LIT_KINDS | VAR_KINDS RELCOORD = { '@0': ['-1','-1'], '@1': ['-1','-2'], '@2': ['-1','-3'], '@3': ['-1','-4'], '@4': ['-1','-5'], '@' : ['-1','+0'], '^' : ['-1','end','+0'], } # Atomic token class Tok: def __init__(self, kind: str, src: str, canon: Optional[str] = None): self.kind = kind self.src = src canon = src if canon is None else canon self.canon = CANON.get(canon) or canon parser = {'QUO': json.loads, 'EMB': json.loads, 'DEC': float, 'INT': int}.get(kind) self.dat = parser(src) if parser else src def __str__(self): return self.src def __repr__(self): return self.canon def __eq__(self, other): return repr(self) == repr(other) def __hash__(self): return hash(self.src) def __bool__(self): return bool(self.src) TOK_NULL = Tok('NULL', '') # Sequence of tokens class Seq(list[Tok]): opr: Tok = TOK_NULL def __init__(self, *items): super().__init__(items) self.opr = TOK_NULL def __str__(self): return self.opr.src.join([str(t) for t in self if len(str(t)) or t.kind=='HOLD']) def __repr__(self): return self.opr.src.join([repr(t) for t in self]) # Predicate expression class Cell: flag: Seq left: Seq comp: Tok right: Seq padded = False def __init__(self, src: str): self.left = Seq() self.flag = Seq() self.comp = Tok('EQL', '', '=') self.right = Seq(Tok('WLD', '', '_')) toks = [] for m in CELL_REGEX.finditer(src): kind = m.lastgroup text = m.group() if kind == 'WS': continue if kind == 'MISMATCH': raise Err(f'E_TOK {text!r}') toks.append(Tok(kind, text)) i, n = 0, len(toks) def peek(): return toks[i].kind if i < n else '' def take(): nonlocal i if i >= n: raise Err('E_EOF') t = toks[i] i += 1 return t # FLAGS while peek() in FLAG_KINDS: self.flag.append(take()) # LEFT (prefix MOD) if peek() == 'MOD': self.left.opr = take() self.left.append(Tok('HOLD', '')) t = take() if not t.kind in DAT_KINDS: raise Err('E_TERM_DAT') self.left.append(t) # COMPARATOR if peek() == 'CMP': self.comp = take() if not peek() in DAT_KINDS: raise Err('E_DAT') # RIGHT (values, OR-joined) if peek() in DAT_KINDS: self.right.clear() while peek() in DAT_KINDS: self.right.append(take()) if peek() == 'OR': self.right.opr = take() if not peek() in DAT_KINDS: raise Err('E_OR_TRAIL') if i != n: raise Err(f'E_EXPR_TRAIL {toks[i:]}') # PLACEHOLDER: OVERWRITE WITH YOUR EMBEDDING FUNCTION def vectorize(self, tok: Tok) -> Tok: if tok.kind == 'EMB': return tok if tok.kind not in {'QUO', 'ALN'}: raise Err('E_EMBED') return Tok('EMB', json.dumps([0.1, 0.2])) @property def single(self) -> Tok: return self.right[0] if self.comp.canon == '=' and len(self.right) == 1 else TOK_NULL @property def literal(self) -> Tok: tok = self.single return tok if tok.kind in LIT_KINDS else TOK_NULL def find(self, kind:str) -> Tok: return next((flag for flag in self.flag if flag.kind == kind), TOK_NULL) def bind(self, tok: Tok): if tok not in self.flag: self.flag.append(tok) def __str__(self) -> str: return f"{self.flag}{self.left}{self.comp}{self.right}" def __repr__(self) -> str: return f"{self.flag!r}{self.left!r}{self.comp!r}{self.right!r}" def __bool__(self) -> bool