Yeah, today was the first day where I felt the need to "see" the problem, but once I busted out plot it felt a bit like cheating, because I could get away with a less general but sufficient solution.
All in all, a very fun puzzle; I got to use a macro, and one of my favourite websites from back when I was in school, geometryalgorithms.com, for the winding-number procedure, which I've used in previous Advents to great effect.
Spoilers
(require
(for-syntax
racket/base)
racket/list
racket/match)
(define-match-expander complex
(lambda (stx)
(syntax-case stx ()
[(_ x y) #'(and (app real-part x) (app imag-part y))]))
(lambda (stx)
(syntax-case stx ()
[(_ x y) #'(make-rectangular x y)]
[_ #'make-rectangular])))
(define (string->coordinate s)
(apply complex (map string->number (regexp-split #px"," s))))
(define red-tiles
(with-input-from-file "input.txt"
(lambda ()
(for/list ([line (in-lines)])
(string->coordinate line)))))
(define area
(match-lambda**
[{(complex p.x p.y)
(complex q.x q.y)}
(* (+ (abs (- p.x q.x)) 1)
(+ (abs (- p.y q.y)) 1))]))
(define discriminant
(match-lambda**
[{(complex p.x p.y)
(complex q.x q.y)
(complex o.x o.y)}
(- (* (- q.x p.x) (- o.y p.y))
(* (- o.x p.x) (- q.y p.y)))]))
(define .x real-part)
(define .y imag-part)
(define hull₁ `(,@red-tiles ,(car red-tiles)))
(define hull₂ `(,@(cdr hull₁) ,(car hull₁)))
(define cache (hash))
(define (cache! o wn)
(begin0 wn (set! cache (hash-set cache o wn))))
(define (winding-number o)
(cond
[(hash-ref cache o #false)]
[else
(match-define (complex o.x o.y) o)
(for/fold ([wn 0] #:result (cache! o wn))
([p (in-list hull₁)]
[q (in-list hull₂)])
(define Δ (discriminant p q o))
(match* (p q)
[{(complex p.x p.y)
(complex q.x q.y)}
(cond
[(<= p.y o.y)
(cond
[(and (> Δ 0) (< o.y q.y)) (+ wn 1)]
[else wn])]
[(and (< Δ 0) (<= q.y o.y)) (- wn 1)]
[else wn])]))]))
(define (bisected? around?)
(for/or ([p (in-list hull₁)]
[q (in-list hull₂)])
(around? (+ p (/ (- q p) 2)))))
(define in-polygon?
(compose1 not zero? winding-number))
(struct rectangle (area frame around?)
#:transparent)
(define ϵ 1/2)
(define (bounds . corners)
(define x-max (.x (argmax .x corners)))
(define x-min (.x (argmin .x corners)))
(define y-max (.y (argmax .y corners)))
(define y-min (.y (argmin .y corners)))
(rectangle
(* (+ (- x-max x-min) 1)
(+ (- y-max y-min) 1))
(list
(complex (- x-max ϵ) (- y-max ϵ))
(complex (+ x-min ϵ) (- y-max ϵ))
(complex (+ x-min ϵ) (+ y-min ϵ))
(complex (- x-max ϵ) (+ y-min ϵ)))
(match-lambda
[(complex x y) (and (< x-min x x-max) (< y-min y y-max))])))
; part 1
(for*/fold ([best -inf.0])
([p (in-list red-tiles)]
[q (in-list red-tiles)]
#:unless (= p q))
(max best (area p q)))
; part 2
(for*/fold ([best -inf.0])
([p (in-list red-tiles)]
[q (in-list red-tiles)]
#:unless (= p q)
#:do [(define r (bounds p q))]
#:when (andmap in-polygon? (rectangle-frame r))
#:unless (bisected? (rectangle-around? r)))
(max best (rectangle-area r)))