SPE-2298: Fix crash caused by a numerical issue during testing if a Voronoi vertex is inside a corner of a polygon.

Cherry-picked from prusa3d/PrusaSlicer@669c931b77

Co-authored-by: Lukáš Hejl <hejl.lukas@gmail.com>

src/libslic3r/Geometry.cpp

@@ -839,4 +839,50 @@ TransformationSVD::TransformationSVD(const Transform3d& trafo)
     return curMat;
 }
 
+bool is_point_inside_polygon_corner(const Point &a, const Point &b, const Point &c, const Point &query_point) {
+    // Cast all input points into int64_t to prevent overflows when points are close to max values of coord_t.
+    const Vec2i64 a_i64           = a.cast<int64_t>();
+    const Vec2i64 b_i64           = b.cast<int64_t>();
+    const Vec2i64 c_i64           = c.cast<int64_t>();
+    const Vec2i64 query_point_i64 = query_point.cast<int64_t>();
+
+    // Shift all points to have a base in vertex B.
+    // Then construct normalized vectors to ensure that we will work with vectors with endpoints on the unit circle.
+    const Vec2d ba = (a_i64 - b_i64).cast<double>().normalized();
+    const Vec2d bc = (c_i64 - b_i64).cast<double>().normalized();
+    const Vec2d bq = (query_point_i64 - b_i64).cast<double>().normalized();
+
+    // Points A and C has to be different.
+    assert(ba != bc);
+
+    // Construct a normal for the vector BQ that points to the left side of the vector BQ.
+    const Vec2d bq_left_normal = perp(bq);
+
+    const double proj_a_on_bq_normal = ba.dot(bq_left_normal); // Project point A on the normal of BQ.
+    const double proj_c_on_bq_normal = bc.dot(bq_left_normal); // Project point C on the normal of BQ.
+    if ((proj_a_on_bq_normal > 0. && proj_c_on_bq_normal <= 0.) || (proj_a_on_bq_normal <= 0. && proj_c_on_bq_normal > 0.)) {
+        // Q is between points A and C or lies on one of those vectors (BA or BC).
+
+        // Based on the CCW order of polygons (contours) and order of corner ABC,
+        // when this condition is met, the query point is inside the corner.
+        return proj_a_on_bq_normal > 0.;
+    } else {
+        // Q isn't between points A and C, but still it can be inside the corner.
+
+        const double proj_a_on_bq = ba.dot(bq); // Project point A on BQ.
+        const double proj_c_on_bq = bc.dot(bq); // Project point C on BQ.
+
+        // The value of proj_a_on_bq_normal is the same when we project the vector BA on the normal of BQ.
+        // So we can say that the Q is on the right side of the vector BA when proj_a_on_bq_normal > 0, and
+        // that the Q is on the left side of the vector BA proj_a_on_bq_normal < 0.
+        // Also, the Q is on the right side of the bisector of oriented angle ABC when proj_c_on_bq < proj_a_on_bq, and
+        // the Q is on the left side of the bisector of oriented angle ABC when proj_c_on_bq > proj_a_on_bq.
+
+        // So the Q is inside the corner when one of the following conditions is met:
+        //  * The Q is on the right side of the vector BA, and the Q is on the right side of the bisector of the oriented angle ABC.
+        //  * The Q is on the left side of the vector BA, and the Q is on the left side of the bisector of the oriented angle ABC.
+        return (proj_a_on_bq_normal > 0. && proj_c_on_bq < proj_a_on_bq) || (proj_a_on_bq_normal <= 0. && proj_c_on_bq >= proj_a_on_bq);
+    }
+}
+
 }} // namespace Slic3r::Geometry