Browse code

Change grep to internal

Robert Cranston authored on 22/01/2023 00:33:32
Showing 1 changed files
... ...
@@ -110,9 +110,9 @@ Environment variables:
110 110
     `enums-tree`, `params`, `params-tree`, `audit,` and `audit-tree` commands.
111 111
     It uses the same format (and defaults) as GREP_COLORS, i.e. a
112 112
     colon-separated list of capabilties: `ms` (matching selected), `fn` (file
113
-    name), `ln` (line number), `se` (separators). Added custom capabilities
114
-    are: `ve` (version), `ex` (extension), `un` (unsupported). Defaults to
115
-    `ms=01;31:fn=35:ln=32:se=36:ve=01;34:ex=34:un=01;33`.
113
+    name), `ln` (line and column number), `se` (separators). Added custom
114
+    capabilities are: `ve` (version), `ex` (extension), `un` (unsupported).
115
+    Defaults to `ms=01;31:fn=35:ln=32:se=36:ve=01;34:ex=34:un=01;33`.
116 116
 """
117 117
 
118 118
 
... ...
@@ -125,6 +125,7 @@ import functools
125 125
 import subprocess
126 126
 import shutil
127 127
 import shlex
128
+import fnmatch
128 129
 import urllib.request
129 130
 import docopt
130 131
 from lxml import etree
... ...
@@ -139,6 +140,7 @@ USER_AGENT    = 'Mozilla/5.0'
139 140
 REGEX         = r'\b(gl|GL_)[0-9A-Z][0-9A-Za-z_]+\b'
140 141
 EXCLUDE_DIRS  = ['.?*', '_*']
141 142
 EXCLUDE_FILES = ['README*', 'TODO*']
143
+BINARY_PEEK   = 1024
142 144
 INDENT        = 2
143 145
 ENV_XDG = lambda var, default: (
144 146
     os.environ.get(f'GLREGISTRY_{var}') or
... ...
@@ -297,21 +299,45 @@ def grep(
297 299
     exclude_files=EXCLUDE_FILES,
298 300
     silent=False,
299 301
 ):
300
-    path          = path if path else '.'
301
-    cmd           = ['grep', '-EIrno']
302
-    exclude_dirs  = [f'--exclude-dir={exclude}' for exclude in exclude_dirs]
303
-    exclude_files = [f'--exclude={exclude}'     for exclude in exclude_files]
304
-    process = subprocess.run(
305
-        [*cmd, *exclude_dirs, *exclude_files, regex, path],
306
-        stdout=subprocess.PIPE,
307
-        stderr=subprocess.DEVNULL if silent else None,
308
-        text=True,
309
-    )
310
-    for string in process.stdout.splitlines():
311
-        string           = removeprefix(f'.{os.path.sep}', string)
312
-        file, line, name = string.split(':', 2)
313
-        line             = int(line)
314
-        yield file, line, name
302
+    path = path if path else '.'
303
+    def onerror(error, file=None):
304
+        file = removeprefix(f'.{os.path.sep}', file or error.filename)
305
+        if silent:
306
+            pass
307
+        elif isinstance(error, OSError):
308
+            log(f"{file}: {error.strerror}")
309
+        elif isinstance(error, UnicodeDecodeError):
310
+            log(f"{file}: {error.reason}")
311
+        else:
312
+            log(f"{file}: {error}")
313
+    def exclude(excludes, names):
314
+        names = set(names)
315
+        for exclude in excludes:
316
+            names -= set(fnmatch.filter(names, exclude))
317
+        return sorted(names)
318
+    def grep_file(file):
319
+        try:
320
+            with open(file, 'rb') as f:
321
+                if 0 in f.read(BINARY_PEEK):
322
+                    return
323
+            with open(file, errors='ignore') as f:
324
+                file = removeprefix(f'.{os.path.sep}', file)
325
+                for line, string in enumerate(f):
326
+                    for match in re.finditer(regex, string):
327
+                        column, name = match.start(), match.group()
328
+                        yield file, line+1, column+1, name
329
+        except Exception as error:
330
+            onerror(error, file)
331
+    if os.path.isfile(path):
332
+        for match in grep_file(path):
333
+            yield match
334
+    else:
335
+        for root, dirs, files in os.walk(path, onerror=onerror):
336
+            dirs [:] = exclude(exclude_dirs,  dirs)
337
+            files[:] = exclude(exclude_files, files)
338
+            for file in files:
339
+                for match in grep_file(os.path.join(root, file)):
340
+                    yield match
315 341
 
316 342
 
317 343
 ## Commands
... ...
@@ -551,25 +577,26 @@ def params_tree(xml, group=None):
551 577
 ### `audit_`
552 578
 def audit_(xml, path=None):
553 579
     audit_ = collections.defaultdict(lambda: collections.defaultdict(list))
554
-    for file, line, name in grep(path):
580
+    for file, line, column, name in grep(path):
555 581
         supports_ = supports(xml, name)
556 582
         if not supports_:
557 583
             supports_ = ['UNSUPPORTED']
558
-        audit_[tuple(supports_)][name].append([file, line])
584
+        audit_[tuple(supports_)][name].append([file, line, column])
559 585
     return audit_
560 586
 
561 587
 
562 588
 ### `audit`
563 589
 def audit(xml, path=None):
564
-    for file, line, supports, name in sorted(
565
-        [file, line, supports, name]
566
-        for supports, names  in audit_(xml, path).items()
567
-        for name, locations  in names.items()
568
-        for file, line       in locations
590
+    for file, line, column, supports, name in sorted(
591
+        [file, line, column, supports, name]
592
+        for supports, names    in audit_(xml, path).items()
593
+        for name, locations    in names.items()
594
+        for file, line, column in locations
569 595
     ):
570 596
         yield indentjoin(0, ':', [
571 597
             color('fn', file),
572 598
             color('ln', line),
599
+            color('ln', column),
573 600
             indentjoin(0, ',', color_supports(supports)),
574 601
             color('ms', name),
575 602
         ])
... ...
@@ -581,10 +608,11 @@ def audit_tree(xml, path=None):
581 608
         yield indentjoin(0, ',', color_supports(supports))
582 609
         for name, locations in sorted(names.items()):
583 610
             yield indentjoin(1, '', [color('ms', name)])
584
-            for file, line in sorted(locations):
611
+            for file, line, column in sorted(locations):
585 612
                 yield indentjoin(2, ':', [
586 613
                     color('fn', file),
587 614
                     color('ln', line),
615
+                    color('ln', column),
588 616
                 ])
589 617
 
590 618
 
Browse code

Change URL

Robert Cranston authored on 28/01/2023 02:32:52
Showing 1 changed files
... ...
@@ -132,7 +132,7 @@ from lxml import etree
132 132
 
133 133
 ## Constants
134 134
 REFPAGES_URL  = 'https://registry.khronos.org/OpenGL-Refpages/'
135
-REGISTRY_URL  = 'https://registry.khronos.org/OpenGL/'
135
+REGISTRY_URL  = 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/'
136 136
 REFPAGES_GIT  = 'https://github.com/KhronosGroup/OpenGL-Refpages'
137 137
 XML_PATH      = 'xml/gl.xml'
138 138
 USER_AGENT    = 'Mozilla/5.0'
Browse code

Change user agent

Robert Cranston authored on 27/04/2022 13:33:50
Showing 1 changed files
... ...
@@ -135,6 +135,7 @@ REFPAGES_URL  = 'https://registry.khronos.org/OpenGL-Refpages/'
135 135
 REGISTRY_URL  = 'https://registry.khronos.org/OpenGL/'
136 136
 REFPAGES_GIT  = 'https://github.com/KhronosGroup/OpenGL-Refpages'
137 137
 XML_PATH      = 'xml/gl.xml'
138
+USER_AGENT    = 'Mozilla/5.0'
138 139
 REGEX         = r'\b(gl|GL_)[0-9A-Z][0-9A-Za-z_]+\b'
139 140
 EXCLUDE_DIRS  = ['.?*', '_*']
140 141
 EXCLUDE_FILES = ['README*', 'TODO*']
... ...
@@ -632,6 +633,9 @@ def refs_all(name):
632 633
 
633 634
 ## Main
634 635
 def main():
636
+    opener = urllib.request.build_opener()
637
+    opener.addheaders = [('User-Agent', USER_AGENT)]
638
+    urllib.request.install_opener(opener)
635 639
     args = docopt.docopt(__doc__)
636 640
     if args['xml']:           edit(xml          ())
637 641
     if args['xml-path']:      page(xml          ())
Browse code

Change enum to be number format agnostic

Robert Cranston authored on 16/09/2023 17:40:57
Showing 1 changed files
... ...
@@ -57,7 +57,7 @@ Commands:
57 57
   value <enum>
58 58
     Print the value of <enum>.
59 59
   enum <value>
60
-    Print the enum(s) that has the given <value>, using exact string matching.
60
+    Print the enum(s) that has the given <value>.
61 61
   supports <name>
62 62
     Print the OpenGL version or extension required to use <name>.
63 63
   names [<support>]
... ...
@@ -411,7 +411,14 @@ def value(xml, enum):
411 411
 
412 412
 ### `enum`
413 413
 def enum(xml, value):
414
-    enum = xml.xpath(f"{ENUMS}/enum[@value='{value}']/@name")
414
+    def conv(s):
415
+        return int(s, 16 if s.startswith('0x') else 10)
416
+    value = conv(value)
417
+    enum = (
418
+        enum.get('name')
419
+        for enum in xml.xpath(f"{ENUMS}/enum")
420
+        if conv(enum.get('value')) == value
421
+    )
415 422
     return sorted(enum, key=key)
416 423
 
417 424
 
Browse code

Memoize

Robert Cranston authored on 15/01/2023 09:25:59
Showing 1 changed files
... ...
@@ -121,6 +121,7 @@ import os
121 121
 import sys
122 122
 import collections
123 123
 import re
124
+import functools
124 125
 import subprocess
125 126
 import shutil
126 127
 import shlex
... ...
@@ -415,6 +416,7 @@ def enum(xml, value):
415 416
 
416 417
 
417 418
 ### `supports`
419
+@functools.cache
418 420
 def supports(xml, name, use_aliases=True):
419 421
     category, attrib = CATEGORY_ATTRIBS['EXTENSION']
420 422
     if xml.xpath(f"{category}[@{attrib}='{name}']"):
Browse code

Add colors

Robert Cranston authored on 11/01/2023 14:29:21
Showing 1 changed files
... ...
@@ -105,6 +105,14 @@ Environment variables:
105 105
     is not defined, `pager` if it exists in `$PATH`, else `less` . The value is
106 106
     interpreted by the shell. If the `$LESS` environment variable is unset, it
107 107
     is set to `FR`.
108
+  GLREGISTRY_COLORS
109
+    If standard out is a terminal, the colors used in output of the `enums`,
110
+    `enums-tree`, `params`, `params-tree`, `audit,` and `audit-tree` commands.
111
+    It uses the same format (and defaults) as GREP_COLORS, i.e. a
112
+    colon-separated list of capabilties: `ms` (matching selected), `fn` (file
113
+    name), `ln` (line number), `se` (separators). Added custom capabilities
114
+    are: `ve` (version), `ex` (extension), `un` (unsupported). Defaults to
115
+    `ms=01;31:fn=35:ln=32:se=36:ve=01;34:ex=34:un=01;33`.
108 116
 """
109 117
 
110 118
 
... ...
@@ -150,6 +158,20 @@ CACHE  = ENV_XDG('CACHE',  os.path.join('~', '.cache'))
150 158
 EDITOR = ENV_PRG('EDITOR', 'vi')
151 159
 PAGER  = ENV_PRG('PAGER',  'less')
152 160
 LESS   = os.environ.get('LESS') or 'FR'
161
+COLORS = collections.defaultdict(str, [
162
+    (color.split('=') + [''])[:2]
163
+    for color in
164
+    filter(None, (
165
+        os.environ.get('GLREGISTRY_COLORS') or
166
+        (lambda x, y: ':'.join([x, x and y]))(
167
+            (
168
+                os.environ.get('GREP_COLORS') or
169
+                'ms=01;31:fn=35:ln=32:se=36'
170
+            ),
171
+            've=01;34:ex=34:un=01;33',
172
+        )
173
+    ).split(':'))
174
+])
153 175
 IN    = lambda a, v, s: f"contains(concat('{s}',@{a},'{s}'),'{s}{v}{s}')"
154 176
 MAYBE = lambda a, v:    f"(@{a}='{v}' or not(@{a}))"
155 177
 TYPES    = "/registry/types"
... ...
@@ -208,9 +230,27 @@ def page(lines):
208 230
         sys.stdout.write(lines)
209 231
 
210 232
 
233
+### `color`
234
+def color(capability, string):
235
+    if not sys.stdout.isatty():
236
+        return string
237
+    return f'\x1b[{COLORS[capability]}m{string}\x1b[m'
238
+
239
+
240
+### `color_supports`
241
+def color_supports(supports):
242
+    for support in supports:
243
+        if support == 'UNSUPPORTED' or support.startswith('<'):
244
+            yield color('un', support)
245
+        elif support.startswith('GL_'):
246
+            yield color('ex', support)
247
+        else:
248
+            yield color('ve', support)
249
+
250
+
211 251
 ### `indentjoin`
212 252
 def indentjoin(indent, sep, parts):
213
-    return ' ' * INDENT * indent + sep.join(map(str, parts))
253
+    return ' ' * INDENT * indent + color('se', sep).join(map(str, parts))
214 254
 
215 255
 
216 256
 ### `removeprefix`
... ...
@@ -455,9 +495,9 @@ def enums(xml, group=None):
455 495
 ### `enums_tree`
456 496
 def enums_tree(xml, group=None):
457 497
     for supports, enums in sorted(enums_(xml, group).items()):
458
-        yield indentjoin(0, ',', supports)
498
+        yield indentjoin(0, ',', color_supports(supports))
459 499
         for enum in sorted(enums):
460
-            yield indentjoin(1, '', [enum])
500
+            yield indentjoin(1, '', [color('ms', enum)])
461 501
 
462 502
 
463 503
 ### `params_`
... ...
@@ -489,13 +529,13 @@ def params(xml, group=None):
489 529
 def params_tree(xml, group=None):
490 530
     for (count, param), occurences in sorted(params_(xml, group).items()):
491 531
         yield indentjoin(0, ':', [
492
-            param,
493
-            -count,
532
+            color('ms', param),
533
+            color('ln', -count),
494 534
         ])
495 535
         for supports_, commands in sorted(occurences.items()):
496
-            yield indentjoin(1, ',', supports_)
536
+            yield indentjoin(1, ',', color_supports(supports_))
497 537
             for command in sorted(commands):
498
-                yield indentjoin(2, '', [command])
538
+                yield indentjoin(2, '', [color('fn', command)])
499 539
 
500 540
 
501 541
 ### `audit_`
... ...
@@ -518,23 +558,23 @@ def audit(xml, path=None):
518 558
         for file, line       in locations
519 559
     ):
520 560
         yield indentjoin(0, ':', [
521
-            file,
522
-            line,
523
-            indentjoin(0, ',', supports),
524
-            name
561
+            color('fn', file),
562
+            color('ln', line),
563
+            indentjoin(0, ',', color_supports(supports)),
564
+            color('ms', name),
525 565
         ])
526 566
 
527 567
 
528 568
 ### `audit_tree`
529 569
 def audit_tree(xml, path=None):
530 570
     for supports, names in sorted(audit_(xml, path).items()):
531
-        yield indentjoin(0, ',', supports)
571
+        yield indentjoin(0, ',', color_supports(supports))
532 572
         for name, locations in sorted(names.items()):
533
-            yield indentjoin(1, '', [name])
573
+            yield indentjoin(1, '', [color('ms', name)])
534 574
             for file, line in sorted(locations):
535 575
                 yield indentjoin(2, ':', [
536
-                    file,
537
-                    line,
576
+                    color('fn', file),
577
+                    color('ln', line),
538 578
                 ])
539 579
 
540 580
 
... ...
@@ -573,11 +613,11 @@ def refs(name):
573 613
 ### `refs_all`
574 614
 def refs_all(name):
575 615
     for support, locations in sorted(refs_(name).items()):
576
-        yield indentjoin(0, ',', [support])
616
+        yield indentjoin(0, ',', color_supports([support]))
577 617
         for name_, url in sorted(locations):
578 618
             yield indentjoin(1, ':', [
579
-                name_,
580
-                url,
619
+                color('ms', name_),
620
+                color('fn', url),
581 621
             ])
582 622
 
583 623
 
Browse code

Add vendors

Robert Cranston authored on 11/01/2023 14:29:12
Showing 1 changed files
... ...
@@ -15,7 +15,9 @@ Usage:
15 15
   glregistry exts
16 16
   glregistry exts-download
17 17
   glregistry exts-all       <name>
18
+  glregistry vendors
18 19
   glregistry type           <type>
20
+  glregistry aliases        <enum>
19 21
   glregistry value          <enum>
20 22
   glregistry enum           <value>
21 23
   glregistry supports       <name>
... ...
@@ -46,8 +48,12 @@ Commands:
46 48
     Download all extension specs.
47 49
   exts-all <name>
48 50
     Print all downloaded extensions that mention <name>.
51
+  vendors
52
+    Print all vendor abbreviations.
49 53
   type <type>
50 54
     Print the definition of <type>.
55
+  aliases <enum>
56
+    Print the KHR, ARB, and EXT aliases of <enum>.
51 57
   value <enum>
52 58
     Print the value of <enum>.
53 59
   enum <value>
... ...
@@ -165,8 +171,11 @@ CHANGE_PREFIXES = [
165 171
     [REQUIRE, '' ],
166 172
     [REMOVE,  '<'],
167 173
 ]
174
+VENDORS = ['KHR', 'ARB', 'EXT']
168 175
 KEY_SUBS = [
169
-    *[[f'^{prefix}', f''] for _, prefix in CHANGE_PREFIXES],
176
+    *[[f'^{   prefix}',  f''       ] for _, prefix in CHANGE_PREFIXES],
177
+    *[[f'{    vendor}$', f'{   i}' ] for i, vendor in enumerate(VENDORS)],
178
+    *[[f'^GL_{vendor}_', f'GL_{i}_'] for i, vendor in enumerate(VENDORS)],
170 179
 ]
171 180
 
172 181
 
... ...
@@ -294,7 +303,7 @@ def ext(extension):
294 303
 def exts(xml):
295 304
     category, attrib = CATEGORY_ATTRIBS['EXTENSION']
296 305
     exts = xml.xpath(f"{category}/@{attrib}")
297
-    return sorted(exts)
306
+    return sorted(exts, key=key)
298 307
 
299 308
 
300 309
 ### `exts_download`
... ...
@@ -317,7 +326,13 @@ def exts_all(xml, name):
317 326
         for file, *_ in
318 327
         grep(os.path.join(CACHE, 'extensions'), rf'\b{name}\b', [], [])
319 328
     )
320
-    return sorted(exts_all)
329
+    return sorted(exts_all, key=key)
330
+
331
+
332
+### `vendors`
333
+def vendors(xml):
334
+    vendors = set(extension.split('_')[1] for extension in exts(xml))
335
+    return sorted(vendors, key=key)
321 336
 
322 337
 
323 338
 ### `type`
... ...
@@ -325,6 +340,29 @@ def type(xml, type):
325 340
     return [xml.xpath(f"string({TYPES}/type/name[text()='{type}']/..)")]
326 341
 
327 342
 
343
+### `aliases`
344
+def aliases(xml, name, supports_=None, vendors_=[]):
345
+    if not vendors_:
346
+        vendors_[:] = vendors(xml)
347
+    for vendor in vendors_:
348
+        if name.endswith(f'_{vendor}'):
349
+            return []
350
+    value_  = value(xml, name)
351
+    if not value_:
352
+        return []
353
+    if not supports_:
354
+        supports_ = supports(xml, name, False)
355
+    if not supports_:
356
+        return []
357
+    aliases = []
358
+    # for vendor in vendors_:
359
+    for vendor in VENDORS:
360
+        alias = f'{name}_{vendor}'
361
+        if supports(xml, alias, False) and value(xml, alias) == value_:
362
+            aliases.append(alias)
363
+    return sorted(aliases, key=key)
364
+
365
+
328 366
 ### `value`
329 367
 def value(xml, enum):
330 368
     return xml.xpath(f"{ENUMS}/enum[@name='{enum}']/@value")
... ...
@@ -333,11 +371,11 @@ def value(xml, enum):
333 371
 ### `enum`
334 372
 def enum(xml, value):
335 373
     enum = xml.xpath(f"{ENUMS}/enum[@value='{value}']/@name")
336
-    return sorted(enum)
374
+    return sorted(enum, key=key)
337 375
 
338 376
 
339 377
 ### `supports`
340
-def supports(xml, name):
378
+def supports(xml, name, use_aliases=True):
341 379
     category, attrib = CATEGORY_ATTRIBS['EXTENSION']
342 380
     if xml.xpath(f"{category}[@{attrib}='{name}']"):
343 381
         return ['EXTENSION']
... ...
@@ -348,6 +386,12 @@ def supports(xml, name):
348 386
         for support in
349 387
         xml.xpath(f"{category}/{change}/*[@name='{name}']/../../@{attrib}")
350 388
     ]
389
+    if supports_ and use_aliases:
390
+        supports_.extend(
391
+            support
392
+            for alias   in aliases (xml, name, supports_)
393
+            for support in supports(xml, alias, False)
394
+        )
351 395
     return sorted(supports_, key=key)
352 396
 
353 397
 
... ...
@@ -374,7 +418,7 @@ def names(xml, support=None):
374 418
         for name in
375 419
         xml.xpath(f"{category}{attrib}/{REQUIRE}/*/@name")
376 420
     )
377
-    return sorted(names)
421
+    return sorted(names, key=key)
378 422
 
379 423
 
380 424
 ### `groups`
... ...
@@ -405,7 +449,7 @@ def enums(xml, group=None):
405 449
         for _, enums in enums_(xml, group).items()
406 450
         for enum     in enums
407 451
     ]
408
-    return sorted(enums)
452
+    return sorted(enums, key=key)
409 453
 
410 454
 
411 455
 ### `enums_tree`
... ...
@@ -547,7 +591,9 @@ def main():
547 591
     if args['exts']:          page(exts         (xml_()))
548 592
     if args['exts-download']: page(exts_download(xml_()))
549 593
     if args['exts-all']:      page(exts_all     (xml_(), args['<name>']))
594
+    if args['vendors']:       page(vendors      (xml_()))
550 595
     if args['type']:          page(type         (xml_(), args['<type>']))
596
+    if args['aliases']:       page(aliases      (xml_(), args['<enum>']))
551 597
     if args['value']:         page(value        (xml_(), args['<enum>']))
552 598
     if args['enum']:          page(enum         (xml_(), args['<value>']))
553 599
     if args['supports']:      page(supports     (xml_(), args['<name>']))
Browse code

Add removed

Robert Cranston authored on 20/01/2023 11:41:36
Showing 1 changed files
... ...
@@ -160,6 +160,14 @@ CATEGORY_ATTRIBS = {
160 160
     ],
161 161
 }
162 162
 REQUIRE = f"require[{MAYBE('api','gl')} and {MAYBE('profile','core')}]"
163
+REMOVE  = f"remove[{ MAYBE('api','gl')} and {MAYBE('profile','core')}]"
164
+CHANGE_PREFIXES = [
165
+    [REQUIRE, '' ],
166
+    [REMOVE,  '<'],
167
+]
168
+KEY_SUBS = [
169
+    *[[f'^{prefix}', f''] for _, prefix in CHANGE_PREFIXES],
170
+]
163 171
 
164 172
 
165 173
 ## Helpers
... ...
@@ -203,6 +211,13 @@ def removeprefix(prefix, string):
203 211
     return string
204 212
 
205 213
 
214
+### `key`
215
+def key(item):
216
+    for sub in KEY_SUBS:
217
+        item = re.sub(*sub, item)
218
+    return item
219
+
220
+
206 221
 ### `download`
207 222
 def download(path, exit_on_failure=True):
208 223
     remote = urllib.parse.urljoin(REGISTRY_URL, path)
... ...
@@ -327,12 +342,13 @@ def supports(xml, name):
327 342
     if xml.xpath(f"{category}[@{attrib}='{name}']"):
328 343
         return ['EXTENSION']
329 344
     supports_ = [
330
-        support
345
+        f'{prefix}{support}'
331 346
         for category, attrib in CATEGORY_ATTRIBS.values()
347
+        for change, prefix   in CHANGE_PREFIXES
332 348
         for support in
333
-        xml.xpath(f"{category}/{REQUIRE}/*[@name='{name}']/../../@{attrib}")
349
+        xml.xpath(f"{category}/{change}/*[@name='{name}']/../../@{attrib}")
334 350
     ]
335
-    return sorted(supports_)
351
+    return sorted(supports_, key=key)
336 352
 
337 353
 
338 354
 ### `names`
Browse code

Add refs

Robert Cranston authored on 16/02/2023 02:22:15
Showing 1 changed files
... ...
@@ -27,6 +27,8 @@ Usage:
27 27
   glregistry params-tree    [<group>]
28 28
   glregistry audit          [<path>]
29 29
   glregistry audit-tree     [<path>]
30
+  glregistry refs           <name>
31
+  glregistry refs-all       <name>
30 32
   glregistry -h|--help
31 33
 
32 34
 Commands:
... ...
@@ -78,6 +80,11 @@ Commands:
78 80
     Search files in <path> if given, or the current directory if omitted,
79 81
     recursively for OpenGL API names and print them sorted on support, name,
80 82
     and location, in a tree.
83
+  refs <name>
84
+    Print all URLs of all reference pages with name <name>.
85
+  refs-all <name>
86
+    Print all URLs of all reference pages that mention <name>, sorted on
87
+    support, in a tree.
81 88
 
82 89
 Environment variables:
83 90
   GLREGISTRY_CACHE
... ...
@@ -109,7 +116,9 @@ from lxml import etree
109 116
 
110 117
 
111 118
 ## Constants
119
+REFPAGES_URL  = 'https://registry.khronos.org/OpenGL-Refpages/'
112 120
 REGISTRY_URL  = 'https://registry.khronos.org/OpenGL/'
121
+REFPAGES_GIT  = 'https://github.com/KhronosGroup/OpenGL-Refpages'
113 122
 XML_PATH      = 'xml/gl.xml'
114 123
 REGEX         = r'\b(gl|GL_)[0-9A-Z][0-9A-Za-z_]+\b'
115 124
 EXCLUDE_DIRS  = ['.?*', '_*']
... ...
@@ -469,6 +478,49 @@ def audit_tree(xml, path=None):
469 478
                 ])
470 479
 
471 480
 
481
+### `refs_`
482
+def refs_(name):
483
+    local = os.path.join(CACHE, os.path.basename(REFPAGES_GIT))
484
+    if not os.path.exists(local):
485
+        os.makedirs(os.path.dirname(local), exist_ok=True)
486
+        subprocess.run(['git', 'clone', REFPAGES_GIT, local])
487
+    refs_ = collections.defaultdict(set)
488
+    for file, *_ in grep(local, rf'\b{name}\b', [], [], True):
489
+        file = removeprefix(f'{local}{os.path.sep}', file)
490
+        try:
491
+            support, *_, dir, base = os.path.normpath(file).split(os.path.sep)
492
+        except:
493
+            continue
494
+        if support.startswith('gl') and dir.endswith('html'):
495
+            support   = removeprefix('gl', support)
496
+            name, ext = os.path.splitext(base)
497
+            url       = urllib.parse.urljoin(REFPAGES_URL, file)
498
+            if ext in ['.xml', '.xhtml']:
499
+                refs_[support].add((name, url))
500
+    return refs_
501
+
502
+
503
+### `refs`
504
+def refs(name):
505
+    return sorted(
506
+        url
507
+        for support, locations in refs_(name).items()
508
+        for name_, url         in locations
509
+        if name_ == name
510
+    )
511
+
512
+
513
+### `refs_all`
514
+def refs_all(name):
515
+    for support, locations in sorted(refs_(name).items()):
516
+        yield indentjoin(0, ',', [support])
517
+        for name_, url in sorted(locations):
518
+            yield indentjoin(1, ':', [
519
+                name_,
520
+                url,
521
+            ])
522
+
523
+
472 524
 ## Main
473 525
 def main():
474 526
     args = docopt.docopt(__doc__)
... ...
@@ -491,6 +543,8 @@ def main():
491 543
     if args['params-tree']:   page(params_tree  (xml_(), args['<group>']))
492 544
     if args['audit']:         page(audit        (xml_(), args['<path>']))
493 545
     if args['audit-tree']:    page(audit_tree   (xml_(), args['<path>']))
546
+    if args['refs']:          page(refs         (args['<name>']))
547
+    if args['refs-all']:      page(refs_all     (args['<name>']))
494 548
 
495 549
 
496 550
 if __name__ == '__main__':
Browse code

Add implementation

Robert Cranston authored on 11/03/2022 20:46:26
Showing 1 changed files
1 1
new file mode 100755
... ...
@@ -0,0 +1,497 @@
1
+#!/usr/bin/env python3
2
+
3
+
4
+## Help
5
+"""
6
+glregistry 1.0
7
+
8
+Cache and query the OpenGL registry locally.
9
+
10
+Usage:
11
+  glregistry xml
12
+  glregistry xml-path
13
+  glregistry ext            <extension>
14
+  glregistry ext-path       <extension>
15
+  glregistry exts
16
+  glregistry exts-download
17
+  glregistry exts-all       <name>
18
+  glregistry type           <type>
19
+  glregistry value          <enum>
20
+  glregistry enum           <value>
21
+  glregistry supports       <name>
22
+  glregistry names          [<support>]
23
+  glregistry groups         [<enum>]
24
+  glregistry enums          [<group>]
25
+  glregistry enums-tree     [<group>]
26
+  glregistry params         [<group>]
27
+  glregistry params-tree    [<group>]
28
+  glregistry audit          [<path>]
29
+  glregistry audit-tree     [<path>]
30
+  glregistry -h|--help
31
+
32
+Commands:
33
+  xml
34
+    Download the registry XML and open it with an editor.
35
+  xml-path
36
+    Download the registry XML and print its local path.
37
+  ext <extension>
38
+    Download the <extension> spec and open it with an editor.
39
+  ext-path <extension>
40
+    Download the <extension> spec and print its local path.
41
+  exts
42
+    Print the names of all extension specs.
43
+  exts-download
44
+    Download all extension specs.
45
+  exts-all <name>
46
+    Print all downloaded extensions that mention <name>.
47
+  type <type>
48
+    Print the definition of <type>.
49
+  value <enum>
50
+    Print the value of <enum>.
51
+  enum <value>
52
+    Print the enum(s) that has the given <value>, using exact string matching.
53
+  supports <name>
54
+    Print the OpenGL version or extension required to use <name>.
55
+  names [<support>]
56
+    Print the names introduced by the OpenGL version or extension <support> if
57
+    given, or all names if omitted. The special values VERSION and EXTENSION
58
+    print the names introduced by all versions or all extensions respectively.
59
+  groups [<enum>]
60
+    Print the groups of <enum> if given, or all groups if omitted.
61
+  enums [<group>]
62
+    Print the enums in <group> if given, or all enums if omitted.
63
+  enums-tree [<group>]
64
+    Print the enums in <group> if given, or all enums if omitted, sorted on
65
+    support, in a tree.
66
+  params [<group>]
67
+    Print the parameter names of <group> if given, or all parameter names if
68
+    omitted.
69
+  params-tree [<group>]
70
+    Print the parameter names of <group> if given, or all parameter names if
71
+    omitted, sorted on count, together with the commands sorted on support, in
72
+    a tree.
73
+  audit [<path>]
74
+    Search files in <path> if given, or the current directory if omitted,
75
+    recursively for OpenGL API names and print them sorted on location,
76
+    support, and name, in a list.
77
+  audit-tree [<path>]
78
+    Search files in <path> if given, or the current directory if omitted,
79
+    recursively for OpenGL API names and print them sorted on support, name,
80
+    and location, in a tree.
81
+
82
+Environment variables:
83
+  GLREGISTRY_CACHE
84
+    The directory to cache files in. Defaults to `$XDG_CACHE_HOME/glregistry`
85
+    or, if `$XDG_CACHE_HOME` is not defined, `$HOME/.cache/glregistry`.
86
+  GLREGISTRY_EDITOR
87
+    The editor to use when opening files. Defaults to `$EDITOR` or, if
88
+    `$EDITOR` is not defined, `editor` if it exists in `$PATH`, else `vi`. The
89
+    value is interpreted by the shell.
90
+  GLREGISTRY_PAGER
91
+    The pager to use when viewing output. Defaults to `$PAGER` or, if `$PAGER`
92
+    is not defined, `pager` if it exists in `$PATH`, else `less` . The value is
93
+    interpreted by the shell. If the `$LESS` environment variable is unset, it
94
+    is set to `FR`.
95
+"""
96
+
97
+
98
+## Imports
99
+import os
100
+import sys
101
+import collections
102
+import re
103
+import subprocess
104
+import shutil
105
+import shlex
106
+import urllib.request
107
+import docopt
108
+from lxml import etree
109
+
110
+
111
+## Constants
112
+REGISTRY_URL  = 'https://registry.khronos.org/OpenGL/'
113
+XML_PATH      = 'xml/gl.xml'
114
+REGEX         = r'\b(gl|GL_)[0-9A-Z][0-9A-Za-z_]+\b'
115
+EXCLUDE_DIRS  = ['.?*', '_*']
116
+EXCLUDE_FILES = ['README*', 'TODO*']
117
+INDENT        = 2
118
+ENV_XDG = lambda var, default: (
119
+    os.environ.get(f'GLREGISTRY_{var}') or
120
+    os.path.join(
121
+        (
122
+            os.environ.get(f'XDG_{var}_HOME') or
123
+            os.path.expanduser(default)
124
+        ),
125
+        'glregistry',
126
+    )
127
+)
128
+ENV_PRG = lambda var, default: (
129
+    os.environ.get(f'GLREGISTRY_{var}') or
130
+    os.environ.get(var) or
131
+    shutil.which(var.lower()) or
132
+    default
133
+)
134
+CACHE  = ENV_XDG('CACHE',  os.path.join('~', '.cache'))
135
+EDITOR = ENV_PRG('EDITOR', 'vi')
136
+PAGER  = ENV_PRG('PAGER',  'less')
137
+LESS   = os.environ.get('LESS') or 'FR'
138
+IN    = lambda a, v, s: f"contains(concat('{s}',@{a},'{s}'),'{s}{v}{s}')"
139
+MAYBE = lambda a, v:    f"(@{a}='{v}' or not(@{a}))"
140
+TYPES    = "/registry/types"
141
+ENUMS    = "/registry/enums"
142
+COMMANDS = "/registry/commands"
143
+CATEGORY_ATTRIBS = {
144
+    'VERSION': [
145
+        "/registry/feature[@api='gl']",
146
+        'number',
147
+    ],
148
+    'EXTENSION': [
149
+        f"/registry/extensions/extension[{IN('supported','gl','|')}]",
150
+        'name',
151
+    ],
152
+}
153
+REQUIRE = f"require[{MAYBE('api','gl')} and {MAYBE('profile','core')}]"
154
+
155
+
156
+## Helpers
157
+
158
+
159
+### `log`
160
+def log(*args, **kwargs):
161
+    print(*args, file=sys.stderr, flush=True, **kwargs)
162
+
163
+
164
+### `edit`
165
+def edit(paths):
166
+    if EDITOR and sys.stdout.isatty():
167
+        args = ' '.join([EDITOR, *map(shlex.quote, paths)])
168
+        subprocess.run(args, shell=True)
169
+    else:
170
+        for path in paths:
171
+            with open(path) as f:
172
+                shutil.copyfileobj(f, sys.stdout)
173
+
174
+
175
+### `page`
176
+def page(lines):
177
+    lines = ''.join(f'{line}\n' for line in lines)
178
+    if lines and PAGER and sys.stdout.isatty():
179
+        args = f'LESS={shlex.quote(LESS)} {PAGER}'
180
+        subprocess.run(args, shell=True, text=True, input=lines)
181
+    else:
182
+        sys.stdout.write(lines)
183
+
184
+
185
+### `indentjoin`
186
+def indentjoin(indent, sep, parts):
187
+    return ' ' * INDENT * indent + sep.join(map(str, parts))
188
+
189
+
190
+### `removeprefix`
191
+def removeprefix(prefix, string):
192
+    if string.startswith(prefix):
193
+        return string[len(prefix):]
194
+    return string
195
+
196
+
197
+### `download`
198
+def download(path, exit_on_failure=True):
199
+    remote = urllib.parse.urljoin(REGISTRY_URL, path)
200
+    local  = os.path.join        (CACHE,        path)
201
+    if not os.path.exists(local):
202
+        try:
203
+            log(f"Downloading '{path}' ... ", end='')
204
+            with urllib.request.urlopen(remote) as response:
205
+                os.makedirs(os.path.dirname(local), exist_ok=True)
206
+                with open(local, 'wb') as f:
207
+                    shutil.copyfileobj(response, f)
208
+        except urllib.error.URLError as error:
209
+            log(error.reason)
210
+            if exit_on_failure:
211
+                exit(1)
212
+        else:
213
+            log(response.reason)
214
+    return local
215
+
216
+
217
+### `grep`
218
+def grep(
219
+    path=None,
220
+    regex=REGEX,
221
+    exclude_dirs=EXCLUDE_DIRS,
222
+    exclude_files=EXCLUDE_FILES,
223
+    silent=False,
224
+):
225
+    path          = path if path else '.'
226
+    cmd           = ['grep', '-EIrno']
227
+    exclude_dirs  = [f'--exclude-dir={exclude}' for exclude in exclude_dirs]
228
+    exclude_files = [f'--exclude={exclude}'     for exclude in exclude_files]
229
+    process = subprocess.run(
230
+        [*cmd, *exclude_dirs, *exclude_files, regex, path],
231
+        stdout=subprocess.PIPE,
232
+        stderr=subprocess.DEVNULL if silent else None,
233
+        text=True,
234
+    )
235
+    for string in process.stdout.splitlines():
236
+        string           = removeprefix(f'.{os.path.sep}', string)
237
+        file, line, name = string.split(':', 2)
238
+        line             = int(line)
239
+        yield file, line, name
240
+
241
+
242
+## Commands
243
+
244
+
245
+### `xml_`
246
+def xml_():
247
+    return etree.parse(download(XML_PATH))
248
+
249
+
250
+### `xml`
251
+def xml():
252
+    return [download(XML_PATH)]
253
+
254
+
255
+### `ext_`
256
+def ext_(extension):
257
+    prefix, vendor, name = extension.split('_', 2)
258
+    if prefix != 'GL':
259
+        log("Extension names must start with 'GL_'.")
260
+        exit(1)
261
+    return f'extensions/{vendor}/{vendor}_{name}.txt'
262
+
263
+
264
+### `ext`
265
+def ext(extension):
266
+    return [download(ext_(extension))]
267
+
268
+
269
+### `exts`
270
+def exts(xml):
271
+    category, attrib = CATEGORY_ATTRIBS['EXTENSION']
272
+    exts = xml.xpath(f"{category}/@{attrib}")
273
+    return sorted(exts)
274
+
275
+
276
+### `exts_download`
277
+def exts_download(xml):
278
+    for ext in exts(xml):
279
+        download(ext_(ext), exit_on_failure=False)
280
+    return []
281
+
282
+
283
+### `exts_all`
284
+def exts_all(xml, name):
285
+    # exts_download(xml)
286
+    exts_all = set(
287
+        'GL_' + os.path.splitext(os.path.basename(file))[0]
288
+        for name in set([
289
+            name,
290
+            removeprefix('gl',  name),
291
+            removeprefix('GL_', name),
292
+        ])
293
+        for file, *_ in
294
+        grep(os.path.join(CACHE, 'extensions'), rf'\b{name}\b', [], [])
295
+    )
296
+    return sorted(exts_all)
297
+
298
+
299
+### `type`
300
+def type(xml, type):
301
+    return [xml.xpath(f"string({TYPES}/type/name[text()='{type}']/..)")]
302
+
303
+
304
+### `value`
305
+def value(xml, enum):
306
+    return xml.xpath(f"{ENUMS}/enum[@name='{enum}']/@value")
307
+
308
+
309
+### `enum`
310
+def enum(xml, value):
311
+    enum = xml.xpath(f"{ENUMS}/enum[@value='{value}']/@name")
312
+    return sorted(enum)
313
+
314
+
315
+### `supports`
316
+def supports(xml, name):
317
+    category, attrib = CATEGORY_ATTRIBS['EXTENSION']
318
+    if xml.xpath(f"{category}[@{attrib}='{name}']"):
319
+        return ['EXTENSION']
320
+    supports_ = [
321
+        support
322
+        for category, attrib in CATEGORY_ATTRIBS.values()
323
+        for support in
324
+        xml.xpath(f"{category}/{REQUIRE}/*[@name='{name}']/../../@{attrib}")
325
+    ]
326
+    return sorted(supports_)
327
+
328
+
329
+### `names`
330
+def names(xml, support=None):
331
+    if support in CATEGORY_ATTRIBS.keys():
332
+        category_attribs = [
333
+            [category, ""]
334
+            for category, _ in [CATEGORY_ATTRIBS[support]]
335
+        ]
336
+    elif support:
337
+        category_attribs = [
338
+            [category, f"[@{attrib}='{support}']"]
339
+            for category, attrib in CATEGORY_ATTRIBS.values()
340
+        ]
341
+    else:
342
+        category_attribs = [
343
+            [category, ""]
344
+            for category, _ in CATEGORY_ATTRIBS.values()
345
+        ]
346
+    names = set(
347
+        name
348
+        for category, attrib in category_attribs
349
+        for name in
350
+        xml.xpath(f"{category}{attrib}/{REQUIRE}/*/@name")
351
+    )
352
+    return sorted(names)
353
+
354
+
355
+### `groups`
356
+def groups(xml, enum=None):
357
+    name = f"[@name='{enum}']" if enum else ""
358
+    return sorted(set(
359
+        group
360
+        for groups in xml.xpath(f"{ENUMS}/enum{name}/@group")
361
+        for group  in groups.split(',')
362
+    ))
363
+
364
+
365
+### `enums_`
366
+def enums_(xml, group=None):
367
+    group = f"[{IN('group',group,',')}]" if group else ""
368
+    enums_ = collections.defaultdict(list)
369
+    for enum in xml.xpath(f"{ENUMS}/enum{group}/@name"):
370
+        supports_ = supports(xml, enum)
371
+        if supports_:
372
+            enums_[tuple(supports_)].append(enum)
373
+    return enums_
374
+
375
+
376
+### `enums`
377
+def enums(xml, group=None):
378
+    enums = [
379
+        enum
380
+        for _, enums in enums_(xml, group).items()
381
+        for enum     in enums
382
+    ]
383
+    return sorted(enums)
384
+
385
+
386
+### `enums_tree`
387
+def enums_tree(xml, group=None):
388
+    for supports, enums in sorted(enums_(xml, group).items()):
389
+        yield indentjoin(0, ',', supports)
390
+        for enum in sorted(enums):
391
+            yield indentjoin(1, '', [enum])
392
+
393
+
394
+### `params_`
395
+def params_(xml, group=None):
396
+    group   = f"[@group='{group}']" if group else ""
397
+    counts  = collections.defaultdict(int)
398
+    params_ = collections.defaultdict(lambda: collections.defaultdict(list))
399
+    for xmlcommand in xml.xpath(f"{COMMANDS}/command/param{group}/.."):
400
+        command   = xmlcommand.xpath(f"string(proto/name)")
401
+        supports_ = supports(xml, command)
402
+        if supports_:
403
+            for xmlparam in xmlcommand.xpath(f"param{group}"):
404
+                param = xmlparam.xpath(f"string(name)")
405
+                params_[param][tuple(supports_)].append(command)
406
+                counts[param] -= 1
407
+    return {
408
+        (count, param): params_[param]
409
+        for param, count in counts.items()
410
+    }
411
+
412
+
413
+### `params`
414
+def params(xml, group=None):
415
+    params = [param for (_, param), _ in params_(xml, group).items()]
416
+    return sorted(params)
417
+
418
+
419
+### `params_tree`
420
+def params_tree(xml, group=None):
421
+    for (count, param), occurences in sorted(params_(xml, group).items()):
422
+        yield indentjoin(0, ':', [
423
+            param,
424
+            -count,
425
+        ])
426
+        for supports_, commands in sorted(occurences.items()):
427
+            yield indentjoin(1, ',', supports_)
428
+            for command in sorted(commands):
429
+                yield indentjoin(2, '', [command])
430
+
431
+
432
+### `audit_`
433
+def audit_(xml, path=None):
434
+    audit_ = collections.defaultdict(lambda: collections.defaultdict(list))
435
+    for file, line, name in grep(path):
436
+        supports_ = supports(xml, name)
437
+        if not supports_:
438
+            supports_ = ['UNSUPPORTED']
439
+        audit_[tuple(supports_)][name].append([file, line])
440
+    return audit_
441
+
442
+
443
+### `audit`
444
+def audit(xml, path=None):
445
+    for file, line, supports, name in sorted(
446
+        [file, line, supports, name]
447
+        for supports, names  in audit_(xml, path).items()
448
+        for name, locations  in names.items()
449
+        for file, line       in locations
450
+    ):
451
+        yield indentjoin(0, ':', [
452
+            file,
453
+            line,
454
+            indentjoin(0, ',', supports),
455
+            name
456
+        ])
457
+
458
+
459
+### `audit_tree`
460
+def audit_tree(xml, path=None):
461
+    for supports, names in sorted(audit_(xml, path).items()):
462
+        yield indentjoin(0, ',', supports)
463
+        for name, locations in sorted(names.items()):
464
+            yield indentjoin(1, '', [name])
465
+            for file, line in sorted(locations):
466
+                yield indentjoin(2, ':', [
467
+                    file,
468
+                    line,
469
+                ])
470
+
471
+
472
+## Main
473
+def main():
474
+    args = docopt.docopt(__doc__)
475
+    if args['xml']:           edit(xml          ())
476
+    if args['xml-path']:      page(xml          ())
477
+    if args['ext']:           edit(ext          (args['<extension>']))
478
+    if args['ext-path']:      page(ext          (args['<extension>']))
479
+    if args['exts']:          page(exts         (xml_()))
480
+    if args['exts-download']: page(exts_download(xml_()))
481
+    if args['exts-all']:      page(exts_all     (xml_(), args['<name>']))
482
+    if args['type']:          page(type         (xml_(), args['<type>']))
483
+    if args['value']:         page(value        (xml_(), args['<enum>']))
484
+    if args['enum']:          page(enum         (xml_(), args['<value>']))
485
+    if args['supports']:      page(supports     (xml_(), args['<name>']))
486
+    if args['names']:         page(names        (xml_(), args['<support>']))
487
+    if args['groups']:        page(groups       (xml_(), args['<enum>']))
488
+    if args['enums']:         page(enums        (xml_(), args['<group>']))
489
+    if args['enums-tree']:    page(enums_tree   (xml_(), args['<group>']))
490
+    if args['params']:        page(params       (xml_(), args['<group>']))
491
+    if args['params-tree']:   page(params_tree  (xml_(), args['<group>']))
492
+    if args['audit']:         page(audit        (xml_(), args['<path>']))
493
+    if args['audit-tree']:    page(audit_tree   (xml_(), args['<path>']))
494
+
495
+
496
+if __name__ == '__main__':
497
+    main()