Skip to content

Commit 6e4ed3c

Browse files
kristoff-itkristoff-it
committed
fmt and <script> bug fixes
- formatting a plaintext file (ie no element tags, just text) multiple times in a row would add extra new lines at the bottom of the file - `<script>` had some funky legacy tokenizing states that were not handled correctly, which are now fixed. this is part of the work to ensure that `<script>` element contents is tokenized correctly since now we have an improved JS rendering algorithm
1 parent 1faf4ed commit 6e4ed3c

2 files changed

Lines changed: 70 additions & 71 deletions

File tree

‎src/html/Ast.zig‎

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -720,6 +720,7 @@ pub fn init(
720720
}
721721

722722
if (std.ascii.eqlIgnoreCase("script", name)) {
723+
log.debug("entering script data state", .{});
723724
tokenizer.gotoScriptData();
724725
} else if (rawtext_names.has(name)) {
725726
tokenizer.gotoRawText(name);
@@ -1146,7 +1147,12 @@ pub fn render(ast: Ast, src: []const u8, w: *Writer) !void {
11461147
try w.writeAll(txt);
11471148
break :blk;
11481149
}
1149-
var it = std.mem.splitScalar(u8, txt, '\n');
1150+
const trimmed_txt = std.mem.trim(
1151+
u8,
1152+
txt,
1153+
&std.ascii.whitespace,
1154+
);
1155+
var it = std.mem.splitScalar(u8, trimmed_txt, '\n');
11501156
var first = true;
11511157
var empty_line = false;
11521158
while (it.next()) |raw_line| {
@@ -1158,7 +1164,8 @@ pub fn render(ast: Ast, src: []const u8, w: *Writer) !void {
11581164
if (line.len == 0) {
11591165
if (empty_line) continue;
11601166
empty_line = true;
1161-
if (!first) for (0..indentation) |_| try w.print("\t", .{});
1167+
assert(!first);
1168+
for (0..indentation) |_| try w.print("\t", .{});
11621169
try w.print("\n", .{});
11631170
continue;
11641171
} else empty_line = false;
@@ -1320,7 +1327,12 @@ pub fn render(ast: Ast, src: []const u8, w: *Writer) !void {
13201327
defer zone.end();
13211328
std.debug.assert(direction == .enter);
13221329

1323-
try w.writeAll(current.open.slice(src));
1330+
try w.writeAll(std.mem.trim(
1331+
u8,
1332+
current.open.slice(src),
1333+
&std.ascii.whitespace,
1334+
));
1335+
13241336
last_rbracket = current.open.end;
13251337

13261338
if (current.next_idx != 0) {
@@ -1909,9 +1921,6 @@ test "tight tags inner indentation" {
19091921
}
19101922

19111923
test "bad html" {
1912-
// TODO: handle ast.errors.len != 0
1913-
if (true) return error.SkipZigTest;
1914-
19151924
const case =
19161925
\\<html>
19171926
\\<body>
@@ -1921,8 +1930,7 @@ test "bad html" {
19211930
;
19221931
const ast = try Ast.init(std.testing.allocator, case, .html, false);
19231932
defer ast.deinit(std.testing.allocator);
1924-
1925-
try std.testing.expectFmt(case, "{f}", .{ast.formatter(case)});
1933+
try std.testing.expect(0 != ast.errors.len);
19261934
}
19271935

19281936
test "formatting - simple" {
@@ -2540,4 +2548,3 @@ test "script formatting - single-line while loop" {
25402548
// .out = &out.interface,
25412549
// }, Context.testOne, .{});
25422550
// }
2543-

‎src/html/Tokenizer.zig‎

Lines changed: 54 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -414,7 +414,12 @@ fn next2(self: *Tokenizer, src: []const u8) ?struct {
414414
deferred: ?Token = null,
415415
} {
416416
while (true) {
417-
log.debug("at char: {any}", .{self.state});
417+
log.debug("[{*}] char: '{c}' ({}) state {any}", .{
418+
self,
419+
self.current,
420+
self.idx,
421+
self.state,
422+
});
418423
switch (self.state) {
419424
.text => |state| {
420425
if (!self.consume(src)) {
@@ -752,7 +757,7 @@ fn next2(self: *Tokenizer, src: []const u8) ?struct {
752757
// This is an unexpected-question-mark-instead-of-tag-name parse error. Create a comment token whose data is the empty string. Reconsume in the bogus comment state.
753758
'?' => {
754759
self.idx -= 1;
755-
self.state = .{ .bogus_comment = self.idx };
760+
self.state = .{ .bogus_comment = lbracket };
756761
},
757762
else => |c| if (isAsciiAlpha(c)) {
758763
// ASCII alpha
@@ -1394,7 +1399,6 @@ fn next2(self: *Tokenizer, src: []const u8) ?struct {
13941399
);
13951400
if (is_script) {
13961401
self.state = .{ .before_attribute_name = tag };
1397-
log.debug("111111", .{});
13981402
if (trimmedText(
13991403
state.data_start,
14001404
state.tag_start,
@@ -1445,7 +1449,6 @@ fn next2(self: *Tokenizer, src: []const u8) ?struct {
14451449
},
14461450
};
14471451

1448-
log.debug("222222", .{});
14491452
if (trimmedText(
14501453
state.data_start,
14511454
state.tag_start,
@@ -1484,7 +1487,6 @@ fn next2(self: *Tokenizer, src: []const u8) ?struct {
14841487
);
14851488
if (is_script) {
14861489
self.state = .data;
1487-
log.debug("3333333", .{});
14881490
if (trimmedText(
14891491
state.data_start,
14901492
state.tag_start,
@@ -1629,8 +1631,12 @@ fn next2(self: *Tokenizer, src: []const u8) ?struct {
16291631
},
16301632
// U+003C LESS-THAN SIGN (<)
16311633
// Switch to the script data escaped less-than sign state.
1632-
'<' => self.state = .{
1633-
.script_data_escaped_less_than_sign = state,
1634+
'<' => {
1635+
var new = state;
1636+
new.tag_start = self.idx - 1;
1637+
self.state = .{
1638+
.script_data_escaped_less_than_sign = new,
1639+
};
16341640
},
16351641
// U+0000 NULL
16361642
// This is an unexpected-null-character parse error. Switch to the script data escaped state. Emit a U+FFFD REPLACEMENT CHARACTER character token.
@@ -1705,7 +1711,7 @@ fn next2(self: *Tokenizer, src: []const u8) ?struct {
17051711
},
17061712
// Anything else
17071713
// Switch to the script data escaped state. Emit the current input character as a character token.
1708-
else => self.state = .{ .script_data = state.data_start },
1714+
else => self.state = .{ .script_data_escaped = state },
17091715
}
17101716
},
17111717

@@ -1723,8 +1729,10 @@ fn next2(self: *Tokenizer, src: []const u8) ?struct {
17231729
// Set the temporary buffer to the empty string. Emit a U+003C LESS-THAN SIGN character token. Reconsume in the script data double escape start state.
17241730
'a'...'z', 'A'...'Z' => {
17251731
self.idx -= 1;
1732+
var new = state;
1733+
new.name_start = self.idx;
17261734
self.state = .{
1727-
.script_data_double_escape_start = state,
1735+
.script_data_double_escape_start = new,
17281736
};
17291737
},
17301738
// Anything else
@@ -1789,7 +1797,6 @@ fn next2(self: *Tokenizer, src: []const u8) ?struct {
17891797
);
17901798
if (is_script) {
17911799
self.state = .{ .before_attribute_name = tag };
1792-
log.debug("555555", .{});
17931800
if (trimmedText(
17941801
state.data_start,
17951802
state.tag_start,
@@ -5219,65 +5226,50 @@ fn trimmedText(start: u32, end: u32, src: []const u8) ?Span {
52195226
}
52205227

52215228
test "script single/double escape weirdness" {
5222-
// TODO: Get this test passing
5223-
if (true) return error.SkipZigTest;
5224-
52255229
// case from https://stackoverflow.com/questions/23727025/script-double-escaped-state
5226-
const case =
5227-
\\<script>
5228-
\\<!--script data escaped-->
5229-
\\</script>
5230-
\\
5231-
\\<script>
5232-
\\<!--<script>script data double escaped</script>-->
5230+
const cases: []const []const u8 = &.{
5231+
//<script>
5232+
// \\<!--script data escaped-->
5233+
// \\</script>
5234+
// ,
5235+
//<script>
5236+
// \\<!--<script>script data double escaped</script>-->
5237+
// \\</script>
5238+
// ,
5239+
//<script>
5240+
\\ <!-- //hide from non-JS browsers
5241+
\\ function doSomething() {
5242+
\\ var coolScript = "<script>" + theCodeICopied + "</script>";
5243+
\\ document.write(coolScript);
5244+
\\ }
5245+
\\ // And if you forget to close your comment here, things go funnny
5246+
\\ -->
52335247
\\</script>
5234-
;
5235-
5236-
// TODO: fix also the expected results!
5248+
};
52375249

5238-
var tokenizer: Tokenizer = .{};
5239-
var t = tokenizer.next(case);
5240-
errdefer std.debug.print("t = {any}\n", .{t});
5250+
for (cases) |c| {
5251+
var tokenizer: Tokenizer = .{ .language = .html };
5252+
tokenizer.gotoScriptData();
5253+
var t = tokenizer.next(c);
52415254

5242-
// first half
5243-
{
5244-
try std.testing.expect(t != null);
5245-
try std.testing.expect(t.? == .tag);
5246-
try std.testing.expect(t.?.tag.kind == .start);
5247-
}
5248-
{
5249-
t = tokenizer.next(case);
5250-
try std.testing.expect(t != null);
5251-
try std.testing.expect(t.? == .text);
5252-
}
5253-
{
5254-
t = tokenizer.next(case);
5255-
try std.testing.expect(t != null);
5256-
try std.testing.expect(t.? == .tag);
5257-
try std.testing.expect(t.?.tag.kind == .end);
5258-
}
5255+
errdefer std.debug.print("token = \n{}\n", .{t.?});
5256+
errdefer std.debug.print("CASE = \n{s}\n", .{c});
5257+
errdefer std.debug.print("[{s}]\n", .{c[126..230]});
52595258

5260-
// Second half
5259+
{
5260+
try std.testing.expect(t != null);
5261+
try std.testing.expect(t.? == .text);
5262+
}
5263+
{
5264+
t = tokenizer.next(c);
5265+
try std.testing.expect(t != null);
5266+
try std.testing.expect(t.? == .tag);
5267+
try std.testing.expect(t.?.tag.kind == .end);
5268+
}
52615269

5262-
{
5263-
try std.testing.expect(t != null);
5264-
try std.testing.expect(t.? == .tag);
5265-
try std.testing.expect(t.?.tag.kind == .start);
5266-
}
5267-
{
5268-
t = tokenizer.next(case);
5269-
try std.testing.expect(t != null);
5270-
try std.testing.expect(t.? == .text);
5270+
t = tokenizer.next(c);
5271+
try std.testing.expect(t == null);
52715272
}
5272-
{
5273-
t = tokenizer.next(case);
5274-
try std.testing.expect(t != null);
5275-
try std.testing.expect(t.? == .tag);
5276-
try std.testing.expect(t.?.tag.kind == .end);
5277-
}
5278-
5279-
t = tokenizer.next(case);
5280-
try std.testing.expect(t == null);
52815273
}
52825274

52835275
test "character references" {

0 commit comments

Comments
 (0)