[DRAFT] Turbopack: Always encode turbopack:///[fs]/ urls as valid URLs

Cargo.toml

@@ -305,6 +305,7 @@ turbopack-swc-utils = { path = "turbopack/crates/turbopack-swc-utils" }
 turbopack-test-utils = { path = "turbopack/crates/turbopack-test-utils" }
 turbopack-trace-server = { path = "turbopack/crates/turbopack-trace-server" }
 turbopack-trace-utils = { path = "turbopack/crates/turbopack-trace-utils" }
+turbopack-url = { path = "turbopack/crates/turbopack-url" }
 turbopack-wasm = { path = "turbopack/crates/turbopack-wasm" }
 
 # SWC crates
@@ -398,6 +399,7 @@ owo-colors = "4.2.2"
 parcel_selectors = "0.28.2"
 parking_lot = "0.12.1"
 pathdiff = "0.2.1"
+percent-encoding = "2.3.2"
 petgraph = "0.6.3"
 pin-project-lite = "0.2.9"
 postcard = "1.0.4"

crates/napi/src/next_api/project.rs

@@ -1,4 +1,4 @@
-use std::{borrow::Cow, io::Write, path::PathBuf, sync::Arc, thread, time::Duration};
+use std::{io::Write, path::PathBuf, sync::Arc, thread, time::Duration};
 
 use anyhow::{Context, Result, anyhow, bail};
 use futures_util::TryFutureExt;
@@ -24,7 +24,6 @@ use next_core::tracing_presets::{
     TRACING_NEXT_OVERVIEW_TARGETS, TRACING_NEXT_TARGETS, TRACING_NEXT_TURBO_TASKS_TARGETS,
     TRACING_NEXT_TURBOPACK_TARGETS,
 };
-use once_cell::sync::Lazy;
 use rand::Rng;
 use serde::{Deserialize, Serialize};
 use tokio::{io::AsyncWriteExt, runtime::Handle, time::Instant};
@@ -38,17 +37,17 @@ use turbo_tasks::{
     trace::TraceRawVcs,
 };
 use turbo_tasks_backend::{BackingStorage, db_invalidation::invalidation_reasons};
-use turbo_tasks_fs::{
-    DiskFileSystem, FileContent, FileSystem, FileSystemPath, util::uri_from_file,
-};
-use turbo_unix_path::{get_relative_path_to, sys_to_unix};
+use turbo_tasks_fs::{DiskFileSystem, FileContent, FileSystem, FileSystemPath};
+use turbo_unix_path::sys_to_unix;
 use turbopack_core::{
-    PROJECT_FILESYSTEM_NAME, SOURCE_URL_PROTOCOL,
+    PROJECT_FILESYSTEM_NAME,
     diagnostics::PlainDiagnostic,
     error::PrettyPrintError,
     issue::PlainIssue,
     output::{OutputAsset, OutputAssets},
-    source_map::{OptionStringifiedSourceMap, SourceMap, Token},
+    source_map::{
+        OptionStringifiedSourceMap, SourceMap, Token, utils::turbopack_url_to_fs_relative_file_url,
+    },
     version::{PartialUpdate, TotalUpdate, Update, VersionState},
 };
 use turbopack_ecmascript_hmr_protocol::{ClientUpdateInstruction, Issue, ResourceIdentifier};
@@ -78,9 +77,6 @@ use crate::{
 /// Used by [`benchmark_file_io`]. This is a noisy benchmark, so set the
 /// threshold high.
 const SLOW_FILESYSTEM_THRESHOLD: Duration = Duration::from_millis(100);
-static SOURCE_MAP_PREFIX: Lazy<String> = Lazy::new(|| format!("{SOURCE_URL_PROTOCOL}///"));
-static SOURCE_MAP_PREFIX_PROJECT: Lazy<String> =
-    Lazy::new(|| format!("{SOURCE_URL_PROTOCOL}///[{PROJECT_FILESYSTEM_NAME}]/"));
 
 #[napi(object)]
 #[derive(Clone, Debug)]
@@ -1479,10 +1475,10 @@ pub fn get_source_map_rope_operation(
 pub async fn project_trace_source_operation(
     container: ResolvedVc<ProjectContainer>,
     frame: StackFrame,
-    current_directory_file_url: RcStr,
 ) -> Result<Vc<OptionStackFrame>> {
     let Some(map) =
-        &*SourceMap::new_from_rope_cached(get_source_map_rope(*container, frame.file)).await?
+        &*SourceMap::new_from_rope_cached(get_source_map_rope(*container, frame.file.clone()))
+            .await?
     else {
         return Ok(Vc::cell(None));
     };
@@ -1496,69 +1492,48 @@ pub async fn project_trace_source_operation(
         frame.column.unwrap_or(1).saturating_sub(1),
     );
 
-    let (original_file, line, column, method_name) = match token {
+    let (original_url, line, column, method_name) = match token {
         Token::Original(token) => (
-            match urlencoding::decode(&token.original_file)? {
-                Cow::Borrowed(_) => token.original_file,
-                Cow::Owned(original_file) => RcStr::from(original_file),
-            },
+            token.original_file,
             // JS stack frames are 1-indexed, source map tokens are 0-indexed
             Some(token.original_line + 1),
             Some(token.original_column + 1),
             token.name,
         ),
         Token::Synthetic(token) => {
-            let Some(original_file) = token.guessed_original_file else {
+            let Some(original_url) = token.guessed_original_file else {
                 return Ok(Vc::cell(None));
             };
-            (original_file, None, None, None)
+            (original_url, None, None, None)
         }
     };
 
-    let project_root_uri =
-        uri_from_file(container.project().project_root_path().owned().await?, None).await? + "/";
-    let (file, original_file, is_internal) =
-        if let Some(source_file) = original_file.strip_prefix(&project_root_uri) {
-            // Client code uses file://
-            (
-                RcStr::from(
-                    get_relative_path_to(&current_directory_file_url, &original_file)
-                        // TODO(sokra) remove this to include a ./ here to make it a relative path
-                        .trim_start_matches("./"),
-                ),
-                Some(RcStr::from(source_file)),
-                false,
-            )
-        } else if let Some(source_file) = original_file.strip_prefix(&*SOURCE_MAP_PREFIX_PROJECT) {
-            // Server code uses turbopack:///[project]
-            // TODO should this also be file://?
-            (
-                RcStr::from(
-                    get_relative_path_to(
-                        &current_directory_file_url,
-                        &format!("{project_root_uri}{source_file}"),
-                    )
-                    // TODO(sokra) remove this to include a ./ here to make it a relative path
-                    .trim_start_matches("./"),
-                ),
-                Some(RcStr::from(source_file)),
-                false,
-            )
-        } else if let Some(source_file) = original_file.strip_prefix(&*SOURCE_MAP_PREFIX) {
-            // All other code like turbopack:///[turbopack] is internal code
-            // TODO(veil): Should the protocol be preserved?
-            (RcStr::from(source_file), None, true)
+    let url_schema = original_url.split_once(":").map(|(schema, _)| schema);
+    let (original_url, is_internal) = if let Some(url_schema) = url_schema
+        && url_schema.eq_ignore_ascii_case("turbopack")
+    {
+        if let Some(relative_url) =
+            turbopack_url_to_fs_relative_file_url(&original_url, PROJECT_FILESYSTEM_NAME)
+        {
+            // Server code uses `turbopack:///[project]` urls?
+            // this returns a relative URL, but should it be an absolute file:// url?
+            //
+            // If it's absolute, that conversion should happen outside of the turbo-task, as that
+            // is not safe to store in the persistent cache (not portable across machines)
+            (Some(RcStr::from(relative_url)), false)
         } else {
-            bail!(
-                "Original file ({}) outside project ({})",
-                original_file,
-                project_root_uri
-            )
-        };
+            // All other code like turbopack:///[turbopack] is internal code
+            (None, true)
+        }
+    } else {
+        // Any other URL... Likely a relative or absolute file:// url. Just pass it through.
+        // Client code uses `file://` urls?
+        (Some(RcStr::from(original_url)), false)
+    };
 
     Ok(Vc::cell(Some(StackFrame {
-        file,
-        original_file,
+        file: frame.file,
+        original_file: original_url,
         method_name,
         line,
         column,
@@ -1572,19 +1547,14 @@ pub async fn project_trace_source_operation(
 pub async fn project_trace_source(
     #[napi(ts_arg_type = "{ __napiType: \"Project\" }")] project: External<ProjectInstance>,
     frame: StackFrame,
-    current_directory_file_url: String,
 ) -> napi::Result<Option<StackFrame>> {
     let container = project.container;
     let ctx = &project.turbopack_ctx;
     ctx.turbo_tasks()
         .run(async move {
-            let traced_frame = project_trace_source_operation(
-                container,
-                frame,
-                RcStr::from(current_directory_file_url),
-            )
-            .read_strongly_consistent()
-            .await?;
+            let traced_frame = project_trace_source_operation(container, frame)
+                .read_strongly_consistent()
+                .await?;
             Ok(ReadRef::into_owned(traced_frame))
         })
         // HACK: Don't use `TurbopackInternalError`, this function is race-condition prone (the

crates/next-api/src/project.rs

@@ -682,6 +682,7 @@ impl Project {
         DiskFileSystem::new(rcstr!("output"), self.root_path.clone())
     }
 
+    // TODO: This should not be a turbo-task function as it is not portable across machines.
     #[turbo_tasks::function]
     pub fn dist_dir_absolute(&self) -> Result<Vc<RcStr>> {
         Ok(Vc::cell(

crates/next-core/Cargo.toml

@@ -33,7 +33,7 @@ rustc-hash = { workspace = true }
 react_remove_properties = { workspace = true }
 remove_console = { workspace = true }
 itertools = { workspace = true }
-percent-encoding = "2.3.1"
+percent-encoding = { workspace = true }
 serde_path_to_error = { workspace = true }
 swc_sourcemap = { workspace = true }
 
@@ -80,6 +80,7 @@ turbopack-node = { workspace = true }
 turbopack-nodejs = { workspace = true }
 turbopack-static = { workspace = true }
 turbopack-trace-utils = { workspace = true }
+turbopack-url = { workspace = true }
 
 [features]
 next-font-local = []

crates/next-core/src/next_manifests/client_reference_manifest.rs

@@ -2,6 +2,7 @@ use anyhow::Result;
 use either::Either;
 use indoc::formatdoc;
 use itertools::Itertools;
+use percent_encoding::utf8_percent_encode;
 use rustc_hash::FxHashMap;
 use serde::Serialize;
 use tracing::Instrument;
@@ -19,13 +20,14 @@ use turbopack_core::{
     output::{OutputAsset, OutputAssets, OutputAssetsWithReferenced},
 };
 use turbopack_ecmascript::utils::StringifyJs;
+use turbopack_url::ENCODE_URI_COMPONENT;
 
 use crate::{
     mode::NextMode,
     next_app::ClientReferencesChunks,
     next_client_reference::{ClientReferenceGraphResult, ClientReferenceType},
     next_config::{CrossOriginConfig, NextConfig},
-    next_manifests::{ModuleId, encode_uri_component::encode_uri_component},
+    next_manifests::ModuleId,
     util::NextRuntime,
 };
 
@@ -290,7 +292,9 @@ async fn build_manifest(
                         format!(
                             "{}{}{}",
                             prefix_path,
-                            path.split('/').map(encode_uri_component).format("/"),
+                            path.split('/')
+                                .map(|segment| utf8_percent_encode(segment, ENCODE_URI_COMPONENT))
+                                .format("/"),
                             suffix_path
                         )
                     })

crates/next-core/src/next_manifests/mod.rs

@@ -1,7 +1,6 @@
 //! Type definitions for the Next.js manifest formats.
 
 pub mod client_reference_manifest;
-mod encode_uri_component;
 
 use anyhow::{Context, Result};
 use serde::{Deserialize, Serialize};

packages/next/src/build/swc/types.ts

@@ -241,8 +241,7 @@ export interface Project {
   getSourceMapSync(filePath: string): string | null
 
   traceSource(
-    stackFrame: TurbopackStackFrame,
-    currentDirectoryFileUrl: string
+    stackFrame: TurbopackStackFrame
   ): Promise<TurbopackStackFrame | null>
 
   updateInfoSubscribe(

packages/next/src/server/dev/middleware-turbopack.ts

@@ -43,10 +43,6 @@ async function batchedTraceSource(
   frame: TurbopackStackFrame
 ): Promise<{ frame: IgnorableStackFrame; source: string | null } | undefined> {
   const file = frame.file
-    ? // TODO(veil): Why are the frames sent encoded?
-      decodeURIComponent(frame.file)
-    : undefined
-
   if (!file) return
 
   // For node internals they cannot traced the actual source code with project.traceSource,
@@ -65,9 +61,7 @@ async function batchedTraceSource(
     }
   }
 
-  const currentDirectoryFileUrl = pathToFileURL(process.cwd()).href
-
-  const sourceFrame = await project.traceSource(frame, currentDirectoryFileUrl)
+  const sourceFrame = await project.traceSource(frame)
   if (!sourceFrame) {
     return {
       frame: {

turbopack/crates/turbo-tasks-fs/Cargo.toml

@@ -37,6 +37,7 @@ jsonc-parser = { version = "0.26.3", features = ["serde"] }
 mime = { workspace = true }
 notify = { workspace = true }
 parking_lot = { workspace = true }
+percent-encoding = { workspace = true }
 regex = { workspace = true }
 rustc-hash = { workspace = true }
 serde = { workspace = true, features = ["rc"] }
@@ -50,6 +51,7 @@ turbo-rcstr = { workspace = true }
 turbo-tasks = { workspace = true }
 turbo-tasks-hash = { workspace = true }
 turbo-unix-path = { workspace = true }
+url = { workspace = true }
 urlencoding = { workspace = true }
 
 [dev-dependencies]

turbopack/crates/turbo-tasks-fs/src/lib.rs

@@ -511,6 +511,10 @@ impl DiskFileSystem {
         })
     }
 
+    pub fn root_sys_path(&self) -> &Path {
+        self.inner.root_path()
+    }
+
     pub fn to_sys_path(&self, fs_path: FileSystemPath) -> Result<PathBuf> {
         let path = self.inner.root_path();
         Ok(if fs_path.path.is_empty() {