Next.js app getting logged out after refresh

We created a next.js app with the app router and the Auth0 V4 SDK.

We have configured our middleware:

import { auth0 } from './lib/auth0';

export async function middleware(req) {
  // Auth only. No slug rewriting/canonicals.
  const publicRoutes = [
    '/', '/robots.txt', '/sitemap.xml', '/favicon.ico',
    '/auth/login', '/login-success',
  ];
  return auth0.middleware(req, { publicRoutes });
}

// Run on everything except Next internals, API, and static assets
export const config = {
  matcher: [
    '/((?!_next/static|_next/image|api|.*\\.(?:png|jpg|jpeg|gif|webp|svg|ico|txt|xml)).*)',
  ],
};

and auth0 lib file:

// lib/auth0.js

import { Auth0Client } from "@auth0/nextjs-auth0/server";

// Initialize the Auth0 client 
export const auth0 = new Auth0Client({
  noContentProfileResponseWhenUnauthenticated: true,
  enableParallelTransactions: false,
  authorizationParameters: {
    // In v4, AUTH0_SCOPE and AUTH0_AUDIENCE must be explicitly provided
    scope: process.env.AUTH0_SCOPE,
    audience: process.env.AUTH0_AUDIENCE,
  }
});

However when changing pages or refreshing a page which uses the useUser() method, the user just gets signed out sometimes. Is there something I am missing here?

Thanks for your help!