frustum - How to select and highlight multiple objects with rectangular selection (ribbon) in three.js

Question

Here I'm trying draw a rect using bounding box in mouse drag and hightlight the objects inside the rect. To draw a rect using box3 (bounding box) on mouse down and mouse up and highlight the particular object inside the rect(bounding box). I think I can achieve them with picking the object inside the bounding box which I don't have a clear idea towards it. Here's the fiddle https://jsfiddle.net/mc7dxokr/

var camera, scene, renderer, mesh, material, controls;
        init();
        animate();
        addCubes();
        render();


        function addCubes() {
            var xDistance = 50;
            var zDistance = 30;
            var geometry = new THREE.BoxGeometry(10, 10, 10);
            var material = new THREE.MeshBasicMaterial({ color: 0x00ff44 });

            //initial offset so does not start in middle.
            var xOffset = -80;

            for (var i = 0; i < 4; i++) {
                for (var j = 0; j < 3; j++) {
                    var mesh = new THREE.Mesh(geometry, material);
                    mesh.position.x = (xDistance * i) + xOffset;
                    mesh.position.z = (zDistance * j);
                    scene.add(mesh);
                }
            };
        }

        function init() {
            // Renderer.
            renderer = new THREE.WebGLRenderer();
            //renderer.setPixelRatio(window.devicePixelRatio);
            renderer.setSize(window.innerWidth, window.innerHeight);
            // Add renderer to page
            document.body.appendChild(renderer.domElement);

            // Create camera.
            camera = new THREE.PerspectiveCamera(70, window.innerWidth / window.innerHeight, 1, 1000);
            camera.position.z = 100;

            // Add controls
            controls = new THREE.TrackballControls(camera);
            controls.addEventListener('change', render);
            controls.enabled = false;

            // Create scene.
            scene = new THREE.Scene();

            // Create ambient light and add to scene.
            var light = new THREE.AmbientLight(0x404040); // soft white light
            scene.add(light);

            // Create directional light and add to scene.
            var directionalLight = new THREE.DirectionalLight(0xffffff);
            directionalLight.position.set(1, 1, 1).normalize();
            scene.add(directionalLight);

            // Add listener for window resize.
            window.addEventListener('resize', onWindowResize, false);
        }

        function animate() {
            requestAnimationFrame(animate);
            controls.update();

        }

        function render() {
            renderer.render(scene, camera);
        }

        function onWindowResize() {
            camera.aspect = window.innerWidth / window.innerHeight;
            camera.updateProjectionMatrix();
            renderer.setSize(window.innerWidth, window.innerHeight);
            controls.handleResize();
        }

Answer 1

Two ways of rendering rectangular ribbons on top of 3D are:

• <div> element overlaying webgl canvas (presented here http://output.jsbin.com/tamoce/3/)
• three.js Line rendered from OrthographicCamera (will be presented in my answer below)

DEMO: http://jsfiddle.net/mmalex/40ucrd8g/

What is Frustum, and how it works: https://www.youtube.com/watch?v=KyTaxN2XUyQ

---

Complete solution you will find here, follow my comments in code:

// this is the core of the solution,
// it builds the Frustum object by given camera and mouse coordinates
function updateFrustrum(camera, mousePos0, mousePos1, frustum) {
    let pos0 = new THREE.Vector3(Math.min(mousePos0.x, mousePos1.x), Math.min(mousePos0.y, mousePos1.y));
    let pos1 = new THREE.Vector3(Math.max(mousePos0.x, mousePos1.x), Math.max(mousePos0.y, mousePos1.y));

    // build near and far planes first
    {
        // camera direction IS normal vector for near frustum plane
        // say - plane is looking "away" from you
        let cameraDir = new THREE.Vector3();
        camera.getWorldDirection(cameraDir);

        // INVERTED! camera direction becomes a normal vector for far frustum plane
        // say - plane is "facing you"
        let cameraDirInv = cameraDir.clone().negate();

        // calc the point that is in the middle of the view, and lies on the near plane
        let cameraNear = camera.position.clone().add(cameraDir.clone().multiplyScalar(camera.near));

        // calc the point that is in the middle of the view, and lies on the far plane
        let cameraFar = camera.position.clone().add(cameraDir.clone().multiplyScalar(camera.far));

        // just build near and far planes by normal+point
        frustum.planes[0].setFromNormalAndCoplanarPoint(cameraDir, cameraNear);
        frustum.planes[1].setFromNormalAndCoplanarPoint(cameraDirInv, cameraFar);
    }

    // next 4 planes (left, right, top and bottom) are built by 3 points:
    // camera postion + two points on the far plane
    // each time we build a ray casting from camera through mouse coordinate, 
    // and finding intersection with far plane.
    // 
    // To build a plane we need 2 intersections with far plane.
    // This is why mouse coordinate will be duplicated and 
    // "adjusted" either in vertical or horizontal direction

    // build frustrum plane on the left
    if (true) {
        let ray = new THREE.Ray();
        ray.origin.setFromMatrixPosition(camera.matrixWorld);
        // Here's the example, - we take X coordinate of a mouse, and Y we set to -0.25 and 0.25 
        // values do not matter here, - important that ray will cast two different points to form 
        // the vertically aligned frustum plane.
        ray.direction.set(pos0.x, -0.25, 1).unproject(camera).sub(ray.origin).normalize();
        let far1 = new THREE.Vector3();
        ray.intersectPlane(frustum.planes[1], far1);

        ray.origin.setFromMatrixPosition(camera.matrixWorld);
        // Same as before, making 2nd ray
        ray.direction.set(pos0.x, 0.25, 1).unproject(camera).sub(ray.origin).normalize();
        let far2 = new THREE.Vector3();
        ray.intersectPlane(frustum.planes[1], far2);

        frustum.planes[2].setFromCoplanarPoints(camera.position, far1, far2);
    }

    // build frustrum plane on the right
    if (true) {
        let ray = new THREE.Ray();
        ray.origin.setFromMatrixPosition(camera.matrixWorld);
        ray.direction.set(pos1.x, 0.25, 1).unproject(camera).sub(ray.origin).normalize();
        let far1 = new THREE.Vector3();
        ray.intersectPlane(frustum.planes[1], far1);

        ray.origin.setFromMatrixPosition(camera.matrixWorld);
        ray.direction.set(pos1.x, -0.25, 1).unproject(camera).sub(ray.origin).normalize();
        let far2 = new THREE.Vector3();
        ray.intersectPlane(frustum.planes[1], far2);

        frustum.planes[3].setFromCoplanarPoints(camera.position, far1, far2);
    }

    // build frustrum plane on the top
    if (true) {
        let ray = new THREE.Ray();
        ray.origin.setFromMatrixPosition(camera.matrixWorld);
        ray.direction.set(0.25, pos0.y, 1).unproject(camera).sub(ray.origin).normalize();
        let far1 = new THREE.Vector3();
        ray.intersectPlane(frustum.planes[1], far1);

        ray.origin.setFromMatrixPosition(camera.matrixWorld);
        ray.direction.set(-0.25, pos0.y, 1).unproject(camera).sub(ray.origin).normalize();
        let far2 = new THREE.Vector3();
        ray.intersectPlane(frustum.planes[1], far2);

        frustum.planes[4].setFromCoplanarPoints(camera.position, far1, far2);
    }

    // build frustrum plane on the bottom
    if (true) {
        let ray = new THREE.Ray();
        ray.origin.setFromMatrixPosition(camera.matrixWorld);
        ray.direction.set(-0.25, pos1.y, 1).unproject(camera).sub(ray.origin).normalize();
        let far1 = new THREE.Vector3();
        ray.intersectPlane(frustum.planes[1], far1);

        ray.origin.setFromMatrixPosition(camera.matrixWorld);
        ray.direction.set(0.25, pos1.y, 1).unproject(camera).sub(ray.origin).normalize();
        let far2 = new THREE.Vector3();
        ray.intersectPlane(frustum.planes[1], far2);

        frustum.planes[5].setFromCoplanarPoints(camera.position, far1, far2);
    }
}

// checks if object is inside of given frustum,
// and updates the object material accordingly
function selectObjects(objects, frustum) {
    // each object in array here is essentially a record:
    // {
    //   obj: scene object,
    //   selected: flag,
    //   bbox: object's bounding box in world coordinates
    // }

    for (let key of Object.keys(objects)) {
        // three.js Frustum can not intersect meshes,
        // it can only intersect boxes, spheres (mainly for performance reasons)
        // TODO: // to make it precisely work with complex meshes, 
        // Frustum needs to check Sphere, Box, and then iterate 
        // throuh mesh vertices array (well, I know, this will be slow)
        if (frustum.intersectsBox(objects[key].bbox)) {
            if (!objects[key].selected) {
                objects[key].obj.material = selectedMaterial;
            }
            objects[key].selected = true;
        } else {
            if (objects[key].selected) {
                objects[key].obj.material = defaultMaterial;
            }
            objects[key].selected = false;
        }
    }
}

// == three.js routine starts here == 
// nothing special, just creating a scene

const SHOW_FRUSTUM_PLANES = false;

var renderer;
var controls;

var scene = new THREE.Scene();
var camera = new THREE.PerspectiveCamera(54, window.innerWidth / window.innerHeight, 1, 100);
camera.position.x = 5;
camera.position.y = 5;
camera.position.z = 5;
camera.lookAt(0, 0, 0);

// this camera is used to render selection ribbon
var ocamera = new THREE.OrthographicCamera(window.innerWidth / -2, window.innerWidth / 2, window.innerHeight / 2, window.innerHeight / -2, 0.1, 1000);
scene.add(ocamera);

ocamera.position.x = 0;
ocamera.position.y = 0;
ocamera.position.z = 100; // this does not matter, just far away

ocamera.lookAt(0, 0, 0);
// IMPORTANT, camera and ribbon are in layer#1,
// Here we render by layers, from two different cameras
ocamera.layers.set(1);

renderer = new THREE.WebGLRenderer({
    antialias: true
});
renderer.setSize(window.innerWidth, window.innerHeight);
renderer.setPixelRatio(window.devicePixelRatio);
renderer.setClearColor(new THREE.Color(0xf9f9f9));
document.body.appendChild(renderer.domElement);

controls = new THREE.OrbitControls(camera); // not used, just abandoned it here

// add some lights
var spotLight = new THREE.SpotLight(0xffffff, 2.5, 25, Math.PI / 4);
spotLight.position.set(4, 10, 7);
scene.add(spotLight);

var size = 6;
var divisions = 6;
var gridHelper = new THREE.GridHelper(size, divisions);
scene.add(gridHelper);

// this material is used for normal object state
var defaultMaterial = new THREE.MeshPhongMaterial({
    color: 0x90a090
});

// this material is used for selected object state
var selectedMaterial = new THREE.MeshPhongMaterial({
    color: 0x20ff20
});

var cubes = {};
// generate some random cubes
for (let i = -2; i <= 2; i++) {
    for (let j = -2; j <= 2; j++) {
        let width = 0.25 + Math.random() * 0.25;
        let height = 0.25 + Math.random() * 0.5;
        let length = width + Math.random() * 0.25;

        let cubeGeometry = new THREE.BoxGeometry(length, height, width);
        let cube = new THREE.Mesh(cubeGeometry, defaultMaterial);
        cube.applyMatrix(new THREE.Matrix4().makeTranslation(i, height / 2, j));
        cubeGeometry.computeBoundingBox();
        let bbox = cubeGeometry.boundingBox.clone();
        bbox.applyMatrix4(cube.matrix);
        scene.add(cube);

        cubes[cube.uuid] = {
            obj: cube, // we need to map the object
            selected: false, // to some flag
            bbox: bbox // and remember it's bounding box (to avoid recalculations on each mouse move)
        };
    }
}

// selection ribbon
var material = new THREE.LineBasicMaterial({
    color: 0x900090
});
var geometry = new THREE.Geometry();
geometry.vertices.push(new THREE.Vector3(-1, -1, 0));
geometry.vertices.push(new THREE.Vector3(-1, 1, 0));
geometry.vertices.push(new THREE.Vector3(1, 1, 0));
geometry.vertices.push(new THREE.Vector3(1, -1, 0));
geometry.vertices.push(new THREE.Vector3(-1, -1, 0));
var line = new THREE.Line(geometry, material);
line.layers.set(1); // IMPORTANT, this goes to layer#1, everything else remains in layer#0 by default
line.visible = false;
scene.add(line);

let frustum = new THREE.Frustum();

// this helpers will visualize frustum planes,
// I keep it here for debug reasons
if (SHOW_FRUSTUM_PLANES) {
    let helper0 = new THREE.PlaneHelper(frustum.planes[0], 1, 0xffff00);
    scene.add(helper0);
    let helper1 = new THREE.PlaneHelper(frustum.planes[1], 1, 0xffff00);
    scene.add(helper1);
    let helper2 = new THREE.PlaneHelper(frustum.planes[2], 1, 0xffff00);
    scene.add(helper2);
    let helper3 = new THREE.PlaneHelper(frustum.planes[3], 1, 0xffff00);
    scene.add(helper3);
    let helper4 = new THREE.PlaneHelper(frustum.planes[4], 1, 0xffff00);
    scene.add(helper4);
    let helper5 = new THREE.PlaneHelper(frustum.planes[5], 1, 0xffff00);
    scene.add(helper5);
}

let pos0, pos1; // mouse coordinates

// You find the code for this class here: https://github.com/nmalex/three.js-helpers
var mouse = new RayysMouse(renderer, camera, controls);

// subscribe my helper class, to receive mouse coordinates
// in convenient format
mouse.subscribe(
    function handleMouseDown(pos, sender) {
        // make selection ribbon visible
        line.visible = true;

        // update ribbon shape verts to match the mouse coordinates
        for (let i = 0; i < line.geometry.vertices.length; i++) {
            line.geometry.vertices[i].x = sender.rawCoords.x;
            line.geometry.vertices[i].y = sender.rawCoords.y;
        }
        geometry.verticesNeedUpdate = true;

        // remember where we started
        pos