stb_truetype: fix incorrect antialiasing computation in v2 rasterizer, and handle certain cases where math blew up

stb_truetype.h

@@ -3120,7 +3120,7 @@ static void stbtt__fill_active_edges_new(float *scanline, float *scanline_fill,
                scanline_fill[x] += e->direction * height; // everything right of this pixel is filled
             } else {
                int x,x1,x2;
-               float y_crossing, step, sign, area;
+               float y_crossing, y_final, step, sign, area;
                // covers 2+ pixels
                if (x_top > x_bottom) {
                   // flip scanline vertically; signed area is the same
@@ -3133,28 +3133,39 @@ static void stbtt__fill_active_edges_new(float *scanline, float *scanline_fill,
                   dy = -dy;
                   t = x0, x0 = xb, xb = t;
                }
+               assert(dy >= 0);
+               assert(dx >= 0);
 
                x1 = (int) x_top;
                x2 = (int) x_bottom;
                // compute intersection with y axis at x1+1
                y_crossing = (x1+1 - x0) * dy + y_top;
+               // if x2 is right at the right edge of x1, y_crossing can blow up, github #1057
+               if (y_crossing > y_bottom)
+                  y_crossing = y_bottom;
 
                sign = e->direction;
                // area of the rectangle covered from y0..y_crossing
                area = sign * (y_crossing-sy0);
                // area of the triangle (x_top,y0), (x+1,y0), (x+1,y_crossing)
-               scanline[x1] += area * (1-((x_top - x1)+(x1+1-x1))/2);
+               scanline[x1] += area * (x1+1 - x_top)/2;
 
-               step = sign * dy;
+               // check if final y_crossing is blown up; no test case for this
+               y_final = y_crossing + dy * (x2 - (x1+1)); // advance y by number of steps taken below
+               if (y_final > y_bottom) {
+                  y_final = y_bottom;
+                  dy = (y_final - y_crossing ) / (x2 - (x1+1)); // if denom=0, y_final = y_crossing, so y_final <= y_bottom
+               }
+
+               step = sign * dy * 1; // dy is dy/dx, change in y for every 1 change in x, which is also how much pixel area changes for each step in x
                for (x = x1+1; x < x2; ++x) {
-                  scanline[x] += area + step/2;
+                  scanline[x] += area + step/2; // area of parallelogram is step/2
                   area += step;
                }
-               y_crossing += dy * (x2 - (x1+1));
+               STBTT_assert(STBTT_fabs(area) <= 1.01f); // accumulated error from area += step unless we round step down
-
-               STBTT_assert(STBTT_fabs(area) <= 1.01f);
 
-               scanline[x2] += area + sign * (1-((x2-x2)+(x_bottom-x2))/2) * (sy1-y_crossing);
+               // area of the triangle (x2,y_crossing), (x_bottom,y1), (x2,y1)
+               scanline[x2] += area + sign * (x_bottom - x2)/2 * (sy1-y_crossing);
 
                scanline_fill[x2] += sign * (sy1-sy0);
             }

tests/stb.dsp

@@ -66,7 +66,7 @@ LINK32=link.exe
 # PROP Ignore_Export_Lib 0
 # PROP Target_Dir ""
 # ADD BASE CPP /nologo /W3 /Gm /GX /ZI /Od /D "WIN32" /D "_DEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /GZ /c
-# ADD CPP /nologo /MTd /W3 /GX /Zi /Od /I ".." /D "WIN32" /D "_DEBUG" /D "_CONSOLE" /D "_MBCS" /D "DS_TEST" /FR /FD /GZ /c
+# ADD CPP /nologo /MTd /W3 /GX /Zi /Od /I ".." /D "WIN32" /D "_DEBUG" /D "_CONSOLE" /D "_MBCS" /D "TT_TEST" /FR /FD /GZ /c
 # SUBTRACT CPP /YX
 # ADD BASE RSC /l 0x409 /d "_DEBUG"
 # ADD RSC /l 0x409 /d "_DEBUG"

tests/test_truetype.c

@@ -37,7 +37,7 @@ int main(int argc, char **argv)
 {
    stbtt_fontinfo font;
    unsigned char *bitmap;
-   int w,h,i,j,c = (argc > 1 ? atoi(argv[1]) : 34807), s = (argc > 2 ? atoi(argv[2]) : 32);
+   int w,h,i,j,c = (argc > 1 ? atoi(argv[1]) : '@'), s = (argc > 2 ? atoi(argv[2]) : 32);
 
    //debug();
 
@@ -49,6 +49,25 @@ int main(int argc, char **argv)
    stbtt_InitFont(&font, ttf_buffer, stbtt_GetFontOffsetForIndex(ttf_buffer,0));
 
 #if 0
+   {
+      stbtt__bitmap b;
+      stbtt__point p[2];
+      int wcount[2] = { 2,0 };
+      p[0].x = 0.2f;
+      p[0].y = 0.3f;
+      p[1].x = 3.8f;
+      p[1].y = 0.8f;
+      b.w = 16;
+      b.h = 2;
+      b.stride = 16;
+      b.pixels = malloc(b.w*b.h);
+      stbtt__rasterize(&b, p, wcount, 1, 1, 1, 0, 0, 0, 0, 0, NULL);
+      for (i=0; i < 8; ++i)
+         printf("%f\n", b.pixels[i]/255.0);
+   }
+#endif
+
+#if 1
    {
       static stbtt_pack_context pc;
       static stbtt_packedchar cd[256];
@@ -60,16 +79,14 @@ int main(int argc, char **argv)
    }
 #endif
 
+
+#if 1
    {
       static stbtt_pack_context pc;
       static stbtt_packedchar cd[256];
       static unsigned char atlas[1024*1024];
       unsigned char *data;
 
-      stbtt_PackBegin(&pc, atlas, 1024,1024,1024,1,NULL);
-      stbtt_PackFontRange(&pc, ttf_buffer, 0, 32.0, 'u', 1, cd);
-      stbtt_PackEnd(&pc);
-
       data = stbtt_GetCodepointSDF(&font, stbtt_ScaleForPixelHeight(&font,32.0), 'u', 4, 128, 128/4, &w,&h,&i,&j);
       for (j=0; j < h; ++j) {
          for (i=0; i < w; ++i) {
@@ -77,8 +94,8 @@ int main(int argc, char **argv)
          }
          putchar('\n');
       }
-      return 0;
    }
+#endif
 
 #if 0
    stbtt_BakeFontBitmap(ttf_buffer,stbtt_GetFontOffsetForIndex(ttf_buffer,0), 40.0, temp_bitmap[0],BITMAP_W,BITMAP_H, 32,96, cdata); // no guarantee this fits!
@@ -117,8 +134,16 @@ int main(int argc, char **argv)
    return 0;
 #endif
 
-   bitmap = stbtt_GetCodepointBitmap(&font, 0,stbtt_ScaleForPixelHeight(&font, (float)s), c, &w, &h, 0,0);
+   (void)stbtt_GetCodepointBitmapSubpixel(&font,
+					 0.4972374737262726f,
+					 0.4986416995525360f,
+					 0.2391788959503174f,
+					 0.1752119064331055f,
+					 'd',
+					 &w, &h,
+					 0,0);
 
+   bitmap = stbtt_GetCodepointBitmap(&font, 0,stbtt_ScaleForPixelHeight(&font, (float)s), c, &w, &h, 0,0);
    for (j=0; j < h; ++j) {
       for (i=0; i < w; ++i)
          putchar(" .:ioVM@"[bitmap[j*w+i]>>5]);