Using VRM in R3F: Experiments and Results

Preconditions

Fundamental knowledge of React.js and Three.js
A VRM model

Dependencies

@react-three/fiber
@react-three/drei
@react-three/postprocessing
@pixiv/three-vrm
three

File Tree (Simplified)

📁 alice-in-moonlight
  |
  ├─ 📁 public
  |   └─ 📄 alice.vrm
  |
  ├─ 📁 src
  |   |
  |   ├─ 📁 hooks
  |   |   └─ 📄 use-vrm-loader.ts
  |   |
  |   ├─ 📁 components
  |   |   └─ 📄 VRMRender.tsx
  |   |
  |   ├─ 📁 utils
  |   |   └─ 📄 humanoid.ts
  |   |
  |   ├─ 📁 scenes
  |   |   └─ 📄 Moonlight.tsx
  |   |
  |   └─ 📁 models
  |       └─ 📄 Alice.tsx
  |
  └─ 📄 package.json

STEP 1: Create the `useVRMLoader` Hook for Loading VRM Files.

/hooks/use-vrm-loader.ts

import { VRM, VRMLoaderPlugin } from '@pixiv/three-vrm'
import { useLoader } from '@react-three/fiber'
import {
  GLTFLoader,
  GLTFLoaderPlugin,
  type GLTFParser,
} from 'three/examples/jsm/loaders/GLTFLoader.js'

export const useVRMloader = (assetUrl: string): VRM => {
  return useLoader(GLTFLoader, assetUrl, (loader) => {
    loader.register((parser: GLTFParser): GLTFLoaderPlugin => {
      return new VRMLoaderPlugin(parser)
    })
  }).userData.vrm
}

STEP 2: Create a VRM Renderer Component.

/components/VRMRender.tsx

import { type FC } from 'react'
import { VRM } from '@pixiv/three-vrm'

export interface VRMRenderProps {
  vrm: VRM
}

export const VRMRender: FC<VRMRenderProps> = (props) => {
  const { vrm } = props

  if (!vrm.scene) {
    return null
  }

  return <primitive object={vrm.scene} dispose={null} />
}

STEP 3: Create a VRM Instance and Render it in the Moonlight Scene.

/models/Alice.tsx

import { VRMRender } from '../components/VRMRender'
import { useVRMloader } from '../hooks/use-vrm-loader'

export const Alice = () => {
  const vrm = useVRMloader('/models/alice.vrm')
  return <VRMRender vrm={vrm} />
}

/scenes/Moonlight.tsx

import { Canvas, SceneProps } from '@react-three/fiber'
import { FC, Suspense } from 'react'
import { Center } from '@react-three/drei'
import { Alice } from '../models/Alice'

export const Moonlight: FC<SceneProps> = () => {
  return (
    <Suspense fallback={null}>
      <Canvas
        style={{
          width: '100vw',
          height: '100vh',
        }}
        camera={{
          fov: 20,
          near: 1,
          position: [0, 0, 4],
        }}
        flat={true}
      >
        <Center>
          <Alice />
        </Center>
        <ambientLight intensity={0.475} />
      </Canvas>
    </Suspense>
  )
}

STEP 4: Create a Utility Tool `humanoid`, for Quick Access to VRM Bone Nodes.

/utils/humanoid.ts

import { VRM, VRMHumanBoneName } from '@pixiv/three-vrm'
import { Object3D } from 'three'

export const humanoid = (vrm: VRM) => {
  const getBoneNode = (boneName: VRMHumanBoneName) => {
    return vrm.humanoid.getNormalizedBoneNode(boneName)!
  }

  const boneNodes = Object.values(VRMHumanBoneName).reduce(
    (acc, value) => {
      acc[value] = getBoneNode(value)
      return acc
    },
    {} as Record<VRMHumanBoneName, Object3D>,
  )

  return { boneNodes }
}

STEP 5: Adjust the Pose of the VRM Instance.

/models/Alice.tsx

import { useFrame } from '@react-three/fiber'
import { VRMRender } from '../components/VRMRender'
import { useVRMloader } from '../hooks/use-vrm-loader'
import { VRMExpressionPresetName } from '@pixiv/three-vrm'
import { humanoid } from '../utils/humanoid'

export const Alice = () => {
  const vrm = useVRMloader('/models/alice.vrm')

  const { boneNodes } = humanoid(vrm)

  const {
    leftUpperArm,
    rightUpperArm,
    leftLowerArm,
    rightLowerArm,
    leftHand,
    rightHand,
    head,
    chest,
    hips,
    spine,
    leftLowerLeg,
    rightLowerLeg,
  } = boneNodes

  leftUpperArm.rotation.set(-1, 1, -1)
  rightUpperArm.rotation.set(-1, -1, 1)
  leftLowerArm.rotation.set(0, 2.5, -1.75)
  rightLowerArm.rotation.set(0, -2.5, 1.75)
  leftHand.rotation.set(2, 0, 0)
  rightHand.rotation.set(2, 0, 0)
  head.rotation.set(0.5, 0, 0)
  chest.rotation.set(1, 0, 0)
  hips.rotation.set(-1.5, 0, 0)
  spine.rotation.set(0.25, 0, 0)
  leftLowerLeg.rotation.set(1, 0, 0)
  rightLowerLeg.rotation.set(0.5, 0, 0)

  // Close eyes
  vrm.expressionManager?.setValue(VRMExpressionPresetName.Blink, 1)
  vrm.expressionManager?.update()

  useFrame((_, delta) => {
    vrm.update(delta)
  })

  return <VRMRender vrm={vrm} />
}

STEP 6: Add Ambient Light, Bloom, Floating Effects, and Loader to `Moonlight` Scene.

/scenes/Moonlight.tsx

import { Canvas, SceneProps } from '@react-three/fiber'
import { FC, Suspense } from 'react'
import {
  Bloom,
  DepthOfField,
  EffectComposer,
} from '@react-three/postprocessing'
import { Center, Float, Loader, OrbitControls } from '@react-three/drei'
import { Alice } from '../models/Alice'

export const Moonlight: FC<SceneProps> = () => {
  return (
    <Suspense fallback={<Loader />}>
      <Canvas
        style={{
          width: '100vw',
          height: '100vh',
        }}
        camera={{
          fov: 20,
          near: 1,
          position: [0, 0, 4],
        }}
        flat={true}
      >
        <Center>
          <Float
            scale={1.2}
            speed={1.5}
            position={[0, 0, 0]}
            rotation={[0, 0.6, 0]}
          >
            <Alice />
          </Float>
        </Center>
        <EffectComposer disableNormalPass>
          <Bloom
            luminanceThreshold={0}
            mipmapBlur
            luminanceSmoothing={0.0}
            intensity={6}
          />
          <DepthOfField
            target={[0, 0, 10]}
            focalLength={0.25}
            bokehScale={15}
            height={800}
          />
        </EffectComposer>
        <ambientLight intensity={0.475} />
        <OrbitControls enablePan enableDamping enableZoom />
      </Canvas>
    </Suspense>
  )
}