Implemented ImageMobject with shaders

manimlib/mobject/mobject.py

@@ -5,6 +5,7 @@ import operator as op
 import os
 import random
 import sys
+import moderngl
 
 from colour import Color
 import numpy as np
@@ -34,7 +35,12 @@ class Mobject(Container):
         "color": WHITE,
         "name": None,
         "dim": 3,
-        "target": None,
+        # For shaders
+        "vert_shader_file": "",
+        "geom_shader_file": "",
+        "frag_shader_file": "",
+        "render_primative": moderngl.TRIANGLE_STRIP,
+        "texture_path": "",
     }
 
     def __init__(self, **kwargs):
@@ -1118,6 +1124,26 @@ class Mobject(Container):
     def cleanup_from_animation(self):
         pass
 
+    # For shaders
+
+    def get_shader_info_list(self):
+        return [self.get_shader_info()]
+
+    def get_shader_info(self):
+        return {
+            "data": self.get_shader_data(),
+            "vert": self.vert_shader_file,
+            "geom": self.geom_shader_file,
+            "frag": self.frag_shader_file,
+            "render_primative": self.render_primative,
+            "texture_path": self.texture_path,
+        }
+
+    def get_shader_data(self):
+        # To be implemented by subclasses
+        # Must return a structured numpy array
+        pass
+
     # Errors
     def throw_error_if_no_points(self):
         if self.has_no_points():

manimlib/mobject/types/image_mobject.py

@@ -4,130 +4,68 @@ from PIL import Image
 
 from manimlib.constants import *
 from manimlib.mobject.mobject import Mobject
-from manimlib.mobject.shape_matchers import SurroundingRectangle
 from manimlib.utils.bezier import interpolate
-from manimlib.utils.color import color_to_int_rgb
-from manimlib.utils.config_ops import digest_config
 from manimlib.utils.images import get_full_raster_image_path
+from manimlib.utils.iterables import listify
 
 
-class AbstractImageMobject(Mobject):
-    """
-    Automatically filters out black pixels
-    """
+class ImageMobject(Mobject):
     CONFIG = {
-        "height": 2.0,
-        "pixel_array_dtype": "uint8",
+        "height": 4,
+        "opacity": 1,
+        "vert_shader_file": "image_vert.glsl",
+        "frag_shader_file": "image_frag.glsl",
     }
 
-    def get_pixel_array(self):
-        raise Exception("Not implemented")
+    def __init__(self, filename, **kwargs):
+        path = get_full_raster_image_path(filename)
+        self.image = Image.open(path)
+        self.texture_path = path
+        Mobject.__init__(self, **kwargs)
 
-    def set_color(self):
-        # Likely to be implemented in subclasses, but no obgligation
-        pass
+    def init_points(self):
+        self.points = np.array([UL, DL, UR, DR])
+        size = self.image.size
+        self.set_width(2 * size[0] / size[1], stretch=True)
+        self.set_height(self.height)
 
-    def reset_points(self):
-        # Corresponding corners of image are fixed to these 3 points
-        self.points = np.array([
-            UP + LEFT,
-            UP + RIGHT,
-            DOWN + LEFT,
-        ])
-        self.center()
-        h, w = self.get_pixel_array().shape[:2]
-        self.stretch_to_fit_height(self.height)
-        self.stretch_to_fit_width(self.height * w / h)
+        self.im_coords = np.array(
+            [(0, 0), (0, 1), (1, 0), (1, 1)]
+        )
 
-    def copy(self):
-        return self.deepcopy()
+    def init_colors(self):
+        self.set_opacity(self.opacity)
 
+    def get_shader_data(self):
+        dtype = [
+            ('point', np.float32, (3,)),
+            ('im_coords', np.float32, (2,)),
+            ('opacity', np.float32, (1,)),
+        ]
 
-class ImageMobject(AbstractImageMobject):
-    CONFIG = {
-        "invert": False,
-        "image_mode": "RGBA",
-    }
+        data = np.zeros(len(self.points), dtype=dtype)
+        data["point"] = self.points
+        data["im_coords"] = self.im_coords
+        data["opacity"] = self.opacity
 
-    def __init__(self, filename_or_array, **kwargs):
-        digest_config(self, kwargs)
-        if isinstance(filename_or_array, str):
-            path = get_full_raster_image_path(filename_or_array)
-            image = Image.open(path).convert(self.image_mode)
-            self.pixel_array = np.array(image)
-        else:
-            self.pixel_array = np.array(filename_or_array)
-        self.change_to_rgba_array()
-        if self.invert:
-            self.pixel_array[:, :, :3] = 255 - self.pixel_array[:, :, :3]
-        AbstractImageMobject.__init__(self, **kwargs)
+        return data
 
-    def change_to_rgba_array(self):
-        pa = self.pixel_array
-        if len(pa.shape) == 2:
-            pa = pa.reshape(list(pa.shape) + [1])
-        if pa.shape[2] == 1:
-            pa = pa.repeat(3, axis=2)
-        if pa.shape[2] == 3:
-            alphas = 255 * np.ones(
-                list(pa.shape[:2]) + [1],
-                dtype=self.pixel_array_dtype
-            )
-            pa = np.append(pa, alphas, axis=2)
-        self.pixel_array = pa
+    def set_opacity(self, alpha, family=True):
+        opacity = listify(alpha)
+        diff = 4 - len(opacity)
+        opacity += [opacity[-1]] * diff
+        self.opacity = np.array(opacity).reshape((4, 1))
 
-    def get_pixel_array(self):
-        return self.pixel_array
-
-    def set_color(self, color, alpha=None, family=True):
-        rgb = color_to_int_rgb(color)
-        self.pixel_array[:, :, :3] = rgb
-        if alpha is not None:
-            self.pixel_array[:, :, 3] = int(255 * alpha)
-        for submob in self.submobjects:
-            submob.set_color(color, alpha, family)
-        self.color = color
-        return self
-
-    def set_opacity(self, alpha):
-        self.pixel_array[:, :, 3] = int(255 * alpha)
-        return self
+        if family:
+            for sm in self.submobjects:
+                sm.set_opacity(alpha)
 
     def fade(self, darkness=0.5, family=True):
-        self.set_opacity(1 - darkness)
-        super().fade(darkness, family)
+        self.set_opacity(1 - darkness, family)
         return self
 
     def interpolate_color(self, mobject1, mobject2, alpha):
-        assert(mobject1.pixel_array.shape == mobject2.pixel_array.shape)
-        self.pixel_array = interpolate(
-            mobject1.pixel_array, mobject2.pixel_array, alpha
-        ).astype(self.pixel_array_dtype)
-
-# TODO, add the ability to have the dimensions/orientation of this
-# mobject more strongly tied to the frame of the camera it contains,
-# in the case where that's a MovingCamera
-
-
-class ImageMobjectFromCamera(AbstractImageMobject):
-    CONFIG = {
-        "default_display_frame_config": {
-            "stroke_width": 3,
-            "stroke_color": WHITE,
-            "buff": 0,
-        }
-    }
-
-    def __init__(self, camera, **kwargs):
-        self.camera = camera
-        AbstractImageMobject.__init__(self, **kwargs)
-
-    def get_pixel_array(self):
-        return self.camera.get_pixel_array()
-
-    def add_display_frame(self, **kwargs):
-        config = dict(self.default_display_frame_config)
-        config.update(kwargs)
-        self.display_frame = SurroundingRectangle(self, **config)
-        self.add(self.display_frame)
-        return self
+        # TODO, transition between actual images?
+        self.opacity = interpolate(
+            mobject1.opacity, mobject2.opacity, alpha
+        )

manimlib/shaders/image_frag.glsl

@@ -0,0 +1,13 @@
+#version 330
+
+uniform sampler2D Texture;
+
+in vec2 v_im_coords;
+in float v_opacity;
+
+out vec4 frag_color;
+
+void main() {
+    frag_color = texture(Texture, v_im_coords);
+    frag_color.a = v_opacity;
+}

manimlib/shaders/image_vert.glsl

@@ -0,0 +1,23 @@
+#version 330
+
+uniform float scale;
+uniform float aspect_ratio;
+uniform vec3 frame_center;
+
+uniform sampler2D Texture;
+
+in vec3 point;
+in vec2 im_coords;
+in float opacity;
+
+out vec2 v_im_coords;
+out float v_opacity;
+
+// Analog of import for manim only
+#INSERT set_gl_Position.glsl
+
+void main(){
+    v_im_coords = im_coords;
+    v_opacity = opacity;
+    set_gl_Position(point);
+}

manimlib/shaders/simple_vert.glsl

@@ -0,0 +1,15 @@
+#version 330
+
+uniform float scale;
+uniform float aspect_ratio;
+uniform float anti_alias_width;
+uniform vec3 frame_center;
+
+in vec3 point;
+
+// Analog of import for manim only
+#INSERT set_gl_Position.glsl
+
+void main(){
+    set_gl_Position(point);
+}