Prototyping a Self-Hosting Ecosystem

This wiki uses and contains the technology it documents. I made it using a new (smaller) design for Denxi, which allows for fun reflective code.

(module cheating-quine racket/base
  (require "denxi.rkt")
  (define-enclosing module-code)
  (writeln module-code))

Rather than request lower-level dependencies from a catalog, Denxi generates higher-level code above itself. This is important, because I want to make sure a given collection path maps to an exact program, regardless of the Racket installation used. The library below is aware that it could be a submodule OR a top-level module, so it can find its own source code among lexical duplicates in a Racket module tree. That means if you paste two copies of the library code at the end of this post next to one another, they can still recognize themselves.

The wiki in the video is an extension of these concepts. It's an HTTP server that generates new implementations of itself based on changes you make in the DOM, client-side. It also uses HMAC authentication of content, such that the Racket module won't even compile without my secret key.

Not ready to release that code yet, since it's part of an incomplete toolchain. Current plan is to use reader extensions to read modules, then assimilate the module form into a new program iff the current Denxi instance contains a submodule that implements the required module language.

Until that happens, here's a prototype of the core library. Please play with it and let me know what you think.

(module denxi racket/base
  (provide (all-defined-out))
  (require (for-syntax racket/base))

  (define (assimilate dependent-id dependency-id bloom . forms)
    (define dependency (full-bloom bloom))
    (and (with-handlers ([exn:fail? (lambda (e) #f)])
           (and (eq? (car dependency) 'module)
                (eq? (cadr dependency) dependency-id)
                (member (caddr dependency) '(racket/base racket))))
         `(module ,dependent-id ,(caddr dependency)
            (begin ,dependency
                   (require ',dependency-id)
                   (module* main #f (exit (enter absolute-path-to-enclosing-file))))
            . ,forms)))

  (define (enter complete [args (current-command-line-arguments)])
    (let/ec return
      (when (= 0 (vector-length args)) (return 255))
      (unless (absolute-path? complete) (return 254))
      (define relative (vector-ref args 0))
      (unless (relative-path? relative) (return 253))
      (define complete-path (if (path? complete) complete (string->path complete)))
      (define module-path
        (for/fold ([where null] #:result `(submod ,complete-path . ,(reverse where)))
                  ([element (in-list (explode-path relative))])
          (case element
            [(up same) (return 252)]
            [else (cons (string->symbol (path->string element)) where)])))
      (parameterize* ([current-command-line-arguments
                       (apply vector-immutable (cdr (vector->list args)))]
                      [exit-handler return])
        (dynamic-require module-path #f))))

  (define-for-syntax (locate stx accessor)
    (or (accessor stx)
        (raise-syntax-error (object-name accessor) "∄" stx)))

  (define-for-syntax (syntax-complete-path stx)
    (let ([path (locate stx syntax-source)])
      (if (complete-path? path)
          (path->string path)
          (raise-syntax-error 'syntax-complete-path
                              (format "~s ∴ ∄" (format "~a" path))
                              stx))))

  (define-for-syntax (blossom-syntax stx)
    #`(blossom #,(locate stx syntax-position)
               #,(syntax-complete-path stx)))

  (define-syntax (absolute-path-to-enclosing-file stx)
    (syntax-case stx ()
      [id (identifier? #'id) (datum->syntax stx (syntax-complete-path stx))]))

  (define-syntax (define-blossom stx)
    (syntax-case stx ()
      [(_ id)
       (identifier? #'id)
       (with-syntax ([e (blossom-syntax stx)])
         #'(define id e))]))

  (define-syntax (define-bloom stx)
    (with-syntax ([e (blossom-syntax stx)])
      (syntax-case stx ()
        [(_ c    ) #'(define-values (c a p) (e))]
        [(_ c   p) #'(define-values (c a p) (e))]
        [(_ c a p) #'(define-values (c a p) (e))])))

  (define-syntax (define-enclosing stx)
    (define site (locate stx syntax-position))
    (define path (syntax-complete-path stx))
    (syntax-case stx ()
      [(_ c . x)
       (identifier? #'c)
       #`(begin (define p (bloom-parent-values (lambda () ((blossom #,site #,path)))))
                (define c (bloom-child p) . x))]))

  (define-syntax (blossom stx)
    (syntax-case stx ()
      [(id)
       #`(blossom #,(locate stx syntax-position)
                  #,(syntax-complete-path stx))]
      [(id position source)
       (exact-positive-integer? (syntax-e #'position))
       #`(λ () (dynamic-blossom (open-input-file source)
                                (λ (p stx) (equal? position p))
                                (λ () (error 'blossom "∄"))))]))

  (define (dynamic-blossom form start-here? fail)
    (if (input-port? form)
        (with-handlers ([values (λ (e) (close-input-port form) (raise e))])
          (call-with-default-reading-parameterization
           (λ ()
             (port-count-lines! form)
             (define (location)
               (call-with-values (λ () (port-next-location form)) list))
             (let read-loop ([loc (location)])
               (with-handlers ([exn:fail:read?
                                (λ (e)
                                  (let ([next (location)])
                                    (if (equal? loc next)
                                        (fail)
                                        (read-loop next))))])
                 (define stx (read-syntax (object-name form) form))
                 (if (syntax? stx)
                     (dynamic-blossom stx
                                      start-here?
                                      (λ () (read-loop (location))))
                     (begin (close-input-port form)
                            (fail))))))))
        (let/ec return
          (define (top-values) (values (syntax->datum form) #f #f))
          (let search ([parent form] [parent-values top-values])
            (define (descend child adjacency)
              (search child
                      (λ ()
                        (values
                         (if (syntax? child)
                             (syntax->datum child)
                             child)
                         adjacency
                         parent-values))))
            (cond [(syntax? parent)
                   (let ([posn (syntax-position parent)])
                     (if (start-here? posn parent)
                         (call-with-values parent-values return)
                         (let ([span (syntax-span parent)])
                           (unless (and span (< (+ (syntax-position parent) span) posn))
                             (search (syntax-e parent) parent-values)))))]
                  [(list? parent)
                   (for ([(child index) (in-indexed (in-list parent))])
                     (descend child index))]
                  [(pair? parent) ; for improper lists
                   (descend (car parent) car)
                   (descend (cdr parent) cdr)]))
          (fail))))

  (define-blossom denxi-bloom)

  (define (denxi? v)
    (with-handlers ([exn:fail? (λ (e) #f)])
      (and (eq? 'module (car v)) (eq? 'denxi (cadr v)) (eq? 'racket/base (caddr v)))))

  (define bloom-loop
    (case-lambda [(? bloom)
                  (call-with-values bloom (λ (c a p) (bloom-loop ? c a p)))]
                 [(? c a p)
                  (if (or (not p) (? c a p))
                      (values c a p)
                      (bloom-loop ? p))]))

  (define full-bloom
    (case-lambda [(bloom) (call-with-values bloom full-bloom)]
                 [(c a p) (if p (full-bloom p) c)]))

  (define (blossom-child c a p) c)
  (define (blossom-adjacency c a p) a)
  (define (blossom-parent-values c a p) p)

  (define (bloom-child bloom) (call-with-values bloom blossom-child))
  (define (bloom-adjacency bloom) (call-with-values bloom blossom-adjacency))
  (define (bloom-parent-values bloom) (call-with-values bloom blossom-parent-values)))
2 Likes

Denxi is a very cool project.

For anyone else curious about self-hosting and related ideas, I found these resources interesting and helpful:

Reflections on Trusting Trust
Bootstrapping and self-hosting

Sage, I know you've mentioned that Denxi could be developed into an alternative to something like raco. Is there a chance it could be an alternative to something like Homebrew as well? Guix for non-Linux systems? Though, I assume you'd start somewhere small and grow from there, bootstrapping the bootstrapped bootstrapper, as it were :slight_smile:

Buffalo buffalo,

2 Likes

I have to be careful here, because I'm experimenting with two designs for Denxi and need to compare both as I answer questions!

Denxi as alternative to ...

I don't think Denxi is an alternative to anything yet, because I don't know of any other program that does what it does. The edition on Denxi currently published on GitHub abstracts over package managers, so something like a Homebrew alternative would be something you write with Denxi.

I assume you'd start somewhere small and grow from there, bootstrapping the bootstrapped bootstrapper, as it were

The edition of Denxi in this thread is an experimental self-hosted approach, and I expect it would play this role.

1 Like