Traceback (most recent call last): File…
- User
Traceback (most recent call last): File "/cs/labs/oabend/shachar.don/repo/self_reflection/dataset_skimming.py", line 17, in dataset = load_share_lm_dataset() ^^^^^^^^^^^^^^^^^^^^^^^ File "/cs/labs/oabend/shachar.don/repo/self_reflection/run_open_router.py", line 134, in load_share_lm_dataset dataset = load_dataset("shachardon/ShareLM", cache_dir="/cs/labs/oabend/shachar.don/hf_cache") ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/cs/labs/oabend/shachar.don/self_env/lib/python3.11/site-packages/datasets/load.py", line 2084, in load_dataset builder_instance.download_and_prepare( File "/cs/labs/oabend/shachar.don/self_env/lib/python3.11/site-packages/datasets/builder.py", line 925, in download_and_prepare self._download_and_prepare( File "/cs/labs/oabend/shachar.don/self_env/lib/python3.11/site-packages/datasets/builder.py", line 979, in _download_and_prepare split_generators = self._split_generators(dl_manager, **split_generators_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/cs/labs/oabend/shachar.don/self_env/lib/python3.11/site-packages/datasets/packaged_modules/parquet/parquet.py", line 49, in _split_generators data_files = dl_manager.download_and_extract(self.config.data_files) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/cs/labs/oabend/shachar.don/self_env/lib/python3.11/site-packages/datasets/download/download_manager.py", line 326, in download_and_extract return self.extract(self.download(url_or_urls)) ^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/cs/labs/oabend/shachar.don/self_env/lib/python3.11/site-packages/datasets/download/download_manager.py", line 159, in download downloaded_path_or_paths = map_nested( ^^^^^^^^^^^ File "/cs/labs/oabend/shachar.don/self_env/lib/python3.11/site-packages/datasets/utils/py_utils.py", line 504, in map_nested mapped = [ ^ File "/cs/labs/oabend/shachar.don/self_env/lib/python3.11/site-packages/datasets/utils/py_utils.py", line 505, in map_nested( File "/cs/labs/oabend/shachar.don/self_env/lib/python3.11/site-packages/datasets/utils/py_utils.py", line 521, in map_nested mapped = [ ^ File "/cs/labs/oabend/shachar.don/self_env/lib/python3.11/site-packages/datasets/utils/py_utils.py", line 522, in _single_map_nested((function, obj, batched, batch_size, types, None, True, None)) File "/cs/labs/oabend/shachar.don/self_env/lib/python3.11/site-packages/datasets/utils/py_utils.py", line 390, in _single_map_nested return [mapped_item for batch in iter_batched(data_struct, batch_size) for mapped_item in function(batch)] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/cs/labs/oabend/shachar.don/self_env/lib/python3.11/site-packages/datasets/utils/py_utils.py", line 390, in return [mapped_item for batch in iter_batched(data_struct, batch_size) for mapped_item in function(batch)] ^^^^^^^^^^^^^^^ File "/cs/labs/oabend/shachar.don/self_env/lib/python3.11/site-packages/datasets/download/download_manager.py", line 206, in _download_batched return thread_map( ^^^^^^^^^^^ File "/cs/labs/oabend/shachar.don/self_env/lib/python3.11/site-packages/tqdm/contrib/concurrent.py", line 69, in thread_map return _executor_map(ThreadPoolExecutor, fn, *iterables, **tqdm_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/cs/labs/oabend/shachar.don/self_env/lib/python3.11/site-packages/tqdm/contrib/concurrent.py", line 51, in _executor_map return list(tqdm_class(ex.map(fn, *iterables, chunksize=chunksize), **kwargs)) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/cs/labs/oabend/shachar.don/self_env/lib/python3.11/site-packages/tqdm/std.py", line 1181, in iter for obj in iterable: File "/usr/lib/python3.11/concurrent/futures/_base.py", line 619, in result_iterator yield _result_or_cancel(fs.pop()) ^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/lib/python3.11/concurrent/futures/_base.py", line 317, in _result_or_cancel return fut.result(timeout) ^^^^^^^^^^^^^^^^^^^ File "/usr/lib/python3.11/concurrent/futures/_base.py", line 456, in result return self.__get_result() ^^^^^^^^^^^^^^^^^^^ File "/usr/lib/python3.11/concurrent/futures/_base.py", line 401, in __get_result raise self._exception File "/usr/lib/python3.11/concurrent/futures/thread.py", line 58, in run result = self.fn(*self.args, **self.kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/cs/labs/oabend/shachar.don/self_env/lib/python3.11/site-packages/datasets/download/download_manager.py", line 229, in _download_single out = cached_path(url_or_filename, download_config=download_config) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/cs/labs/oabend/shachar.don/self_env/lib/python3.11/site-packages/datasets/utils/file_utils.py", line 197, in cached_path ).hf_hub_download( ^^^^^^^^^^^^^^^^ File "/cs/labs/oabend/shachar.don/self_env/lib/python3.11/site-packages/huggingface_hub/utils/_validators.py", line 89, in _inner_fn return fn(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^ File "/cs/labs/oabend/shachar.don/self_env/lib/python3.11/site-packages/huggingface_hub/hf_api.py", line 5363, in hf_hub_download return hf_hub_download( ^^^^^^^^^^^^^^^^ File "/cs/labs/oabend/shachar.don/self_env/lib/python3.11/site-packages/huggingface_hub/utils/_validators.py", line 89, in _inner_fn return fn(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^ File "/cs/labs/oabend/shachar.don/self_env/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 1007, in hf_hub_download return _hf_XXX( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/cs/labs/oabend/shachar.don/self_env/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 1200, in _hf_XXX _download_to_tmp_and_move( File "/cs/labs/oabend/shachar.don/self_env/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 1791, in _download_to_tmp_and_move xet_get( File "/cs/labs/oabend/shachar.don/self_env/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 571, in xet_get download_files( RuntimeError: Data processing error: CAS service error : IO Error: Disk quota exceeded (os error 122)
Process finished with exit code 1
- Assistant
You're hitting the key error at the very bottom: vbnetCopy codeRuntimeError: Data processing error: CAS service error : IO Error: Disk quota exceeded (os error 122)
This means the HuggingFace dataset download is failing because your home directory (or the cache directory you set) ran out of disk quota. Why it happens You're using: pythonCopy codeload_dataset("shachardon/ShareLM", cache_dir="/cs/labs/oabend/shachar.don/hf_cache")
HF first downloads all parquet shards to that cache directory. If the dataset is large (and ShareLM is large), you can easily exceed your lab disk quota.
✅ How to fix Option 1 — Clean the HuggingFace cache Run this: bashCopy coderm -rf ~/.cache/huggingface rm -rf /cs/labs/oabend/shachar.don/hf_cache/*
Be careful: this deletes downloaded model/dataset caches, so they will re-download later.
Option 2 — Point the cache to a project directory with more space If your lab has a large /cs/scratch or project-specific shared folder, do: pythonCopy codeload_dataset( "shachardon/ShareLM", cache_dir="/cs/scratch/shachar/hf_cache" )
Or whatever directory quota you can use. You can also globally set: bashCopy codeexport HF_HOME=/cs/scratch/shachar/hf export HF_DATASETS_CACHE=/cs/scratch/shachar/hf/datasets export HF_METRICS_CACHE=/cs/scratch/shachar/hf/metrics
Option 3 — Disable caching entirely Useful for one-off processing: pythonCopy codeload_dataset( "shachardon/ShareLM", cache_dir="/dev/shm" # RAM disk; only works if dataset fits in memory )
But ShareLM is way too big for RAM.
Option 4 — Stream the dataset instead of downloading This completely avoids local storage: pythonCopy codedataset = load_dataset("shachardon/ShareLM", streaming=True) for x in dataset["train"]: ...
Streaming = no cache, but you lose random access and some transforms. This is the easiest workaround.
📌 Most recommended for your situation Given you're working on a lab server and quotas are very strict: → Use streaming OR redirect cache to a scratch directory. I'd do: pythonCopy codedataset = load_dataset("shachardon/ShareLM", streaming=True)
Or: pythonCopy codeload_dataset("shachardon/ShareLM", cache_dir="/cs/labs/oabend/shared_storage/hf_cache")
(if that directory exists)
If you want, you can paste your df -h or quota output and I’ll suggest the cleanest path on your lab system.
- User
export HF_HOME=/cs/scratch/shachar/hf export HF_DATASETS_CACHE=/cs/scratch/shachar/hf/datasets export HF_METRICS_CACHE=/cs/scratch/shachar/hf/metrics
How can I do that within python?
- Assistant
You can set these environment variables directly inside Python before calling load_dataset. Here's the cleanest way: pythonCopy codeimport os
os.environ["HF_HOME"] = "/cs/scratch/shachar/hf" os.environ["HF_DATASETS_CACHE"] = "/cs/scratch/shachar/hf/datasets" os.environ["HF_METRICS_CACHE"] = "/cs/scratch/shachar/hf/metrics"
from datasets import load_dataset
dataset = load_dataset("shachardon/ShareLM")
Important notes
Set the environment variables before importing or calling anything from huggingface_hub or datasets.
The directories must exist. If not, create them:
pythonCopy codeos.makedirs("/cs/scratch/shachar/hf/datasets", exist_ok=True) os.makedirs("/cs/scratch/shachar/hf/metrics", exist_ok=True)
Optional: verify that HF is using the new location pythonCopy codefrom huggingface_hub import hf_hub_download print(os.environ["HF_HOME"])
If you want, I can also show how to do this per script or per virtualenv automatically.
- User
Is there any chance that hf is storing somewhere else temporally during the download?
- Assistant
Yes — HuggingFace does use additional temporary locations during downloads, and these can easily cause “disk quota exceeded” even if you set HF_HOME and cache dirs. Here are all the places HF may write to:
✅ 1. Temporary directory used during downloads (tempfile.gettempdir()) Before a file is moved into your cache directory, HF downloads it into a temporary file, typically in:
/tmp
or /var/tmp
or whatever $TMPDIR is set to
On many university/lab servers, /tmp has very small quotas. Fix You can redirect the temporary directory inside Python: pythonCopy codeimport os os.environ["TMPDIR"] = "/cs/scratch/shachar/tmp" os.environ["TEMP"] = "/cs/scratch/shachar/tmp" os.environ["TMP"] = "/cs/scratch/shachar/tmp"
Create the folder if needed: pythonCopy codeos.makedirs("/cs/scratch/shachar/tmp", exist_ok=True)
✅ 2. HuggingFace uses a staging directory inside the cache Even inside the HF cache, downloads go to: bashCopy code<cache_dir>/downloads/.incomplete
Then they are moved to final location. If <cache_dir> doesn’t have space → failure. Your cache_dir is /cs/scratch/shachar/hf, so make sure that entire directory has quota.
✅ 3. Datasets library sometimes extracts archives to
/.cache/huggingface Even if you redirect the cache, dataset builder modules (Parquet, JSON, etc.) sometimes fallback to: bashCopy code/.cache/huggingface/datasetsdue to older cached configs or partial downloads. Fix: pythonCopy codeimport os os.environ["HF_HOME"] = "/cs/scratch/shachar/hf" os.environ["HF_DATASETS_CACHE"] = "/cs/scratch/shachar/hf/datasets" os.environ["HF_METRICS_CACHE"] = "/cs/scratch/shachar/hf/metrics" os.environ["HF_HUB_CACHE"] = "/cs/scratch/shachar/hf/hub"
Also remove the old hidden cache: bashCopy coderm -rf ~/.cache/huggingface
✅ 4. Parquet engine temporary files (PyArrow) PyArrow also writes large temp files to $TMPDIR. So you must set TMPDIR.
🧩 Final recommended block (put at very top of your script) pythonCopy codeimport os
Redirect HF caches
os.environ["HF_HOME"] = "/cs/scratch/shachar/hf" os.environ["HF_DATASETS_CACHE"] = "/cs/scratch/shachar/hf/datasets" os.environ["HF_METRICS_CACHE"] = "/cs/scratch/shachar/hf/metrics" os.environ["HF_HUB_CACHE"] = "/cs/scratch/shachar/hf/hub"
Redirect temp storage (very important!)
os.environ["TMPDIR"] = "/cs/scratch/shachar/tmp" os.environ["TEMP"] = "/cs/scratch/shachar/tmp" os.environ["TMP"] = "/cs/scratch/shachar/tmp"
os.makedirs("/cs/scratch/shachar/tmp", exist_ok=True)
from datasets import load_dataset
If you want, I can print out all paths HF is using at runtime to verify none point to your home directory.
- User
I really think that there should be enough space in the dir. Maybe there are older files that stuck it somehow? pointing to the wrong place?
- Assistant
Yes — this is very likely. HuggingFace caching can get messy, and old partial downloads or hidden fallback directories can silently fill up your quota even if your main cache dir has plenty of free space. Let’s systematically check every place HF might be using, and how to clean it.
✅ Step 1 — Verify where HF is actually writing at runtime Add this diagnostic block before load_dataset: pythonCopy codeimport huggingface_hub import datasets import os import tempfile
print("HF_HOME:", os.environ.get("HF_HOME")) print("HF_DATASETS_CACHE:", os.environ.get("HF_DATASETS_CACHE")) print("HF_METRICS_CACHE:", os.environ.get("HF_METRICS_CACHE")) print("HF_HUB_CACHE:", os.environ.get("HF_HUB_CACHE"))
print("HF default cache dir:", huggingface_hub.constants.HF_HOME) print("Datasets default cache:", datasets.config.HF_DATASETS_CACHE) print("TMPDIR:", os.environ.get("TMPDIR")) print("tempfile.gettempdir():", tempfile.gettempdir())
This will reveal if HF or datasets are still falling back to:
~/.cache/huggingface
/tmp
$HOME/.cache
✅ Step 2 — Clean all old HF caches (these fill silently) Run: bashCopy coderm -rf ~/.cache/huggingface rm -rf ~/.cache/huggingface_hub rm -rf ~/.cache/huggingface_modules rm -rf ~/.cache/datasets
Also check: bashCopy codedu -sh ~/.cache
Important: HF may store partial .incomplete files that never get deleted — these often take gigabytes.
✅ Step 3 — Check your lab quota Your lab machine probably uses quota. Try: bashCopy codequota -s
Or: bashCopy codedu -sh /cs/labs/oabend/shachar.don/* du -sh /cs/labs/oabend/shachar.don/
This will show if hidden garbage is sitting somewhere unexpected.
✅ Step 4 — HF has two cache systems Even if you set HF_HOME, datasets may still use its own cache directory unless you set both: pythonCopy codeos.environ["HF_HOME"] = "/cs/scratch/shachar/hf" os.environ["HF_HUB_CACHE"] = "/cs/scratch/shachar/hf/hub" os.environ["HF_DATASETS_CACHE"] = "/cs/scratch/shachar/hf/datasets" os.environ["HF_METRICS_CACHE"] = "/cs/scratch/shachar/hf/metrics"
And you must delete the old caches or datasets will fallback.
✅ Step 5 — Temporary files are the #1 cause of this error Even if your cache dir has space, HF downloads into /tmp/ first, THEN moves the file. If /tmp has a small quota → you hit: nginxCopy codeCAS service error: Disk quota exceeded
Check /tmp usage: bashCopy codedu -sh /tmp/*
Or check mount type: bashCopy codedf -h /tmp
If /tmp is tiny (common on lab servers), set: pythonCopy codeos.environ["TMPDIR"] = "/cs/scratch/shachar/tmp" os.makedirs("/cs/scratch/shachar/tmp", exist_ok=True)
This is often the real fix.
🧪 Want me to help find the exact culprit? If you run: pythonCopy codeimport os, tempfile, huggingface_hub, datasets
print("TMP:", tempfile.gettempdir()) print("HF_HOME:", huggingface_hub.constants.HF_HOME) print("DATASETS:", datasets.config.HF_DATASETS_CACHE)
and paste the output here, I can pinpoint exactly which directory is causing the quota failure.
- User
HF_HOME: /cs/labs/oabend/shachar.don/hf_cache HF_DATASETS_CACHE: /cs/labs/oabend/shachar.don/hf_cache HF_METRICS_CACHE: None HF_HUB_CACHE: /cs/labs/oabend/shachar.don/hf_cache HF default cache dir: /cs/usr/shachar.don/.cache/huggingface Datasets default cache: /cs/usr/shachar.don/.cache/huggingface/datasets TMPDIR: None tempfile.gettempdir(): /tmp HF_HOME: /cs/labs/oabend/shachar.don/hf_cache HF_DATASETS_CACHE: /cs/labs/oabend/shachar.don/hf_cache HF_METRICS_CACHE: None HF_HUB_CACHE: /cs/labs/oabend/shachar.don/hf_cache HF default cache dir: /cs/usr/shachar.don/.cache/huggingface Datasets default cache: /cs/usr/shachar.don/.cache/huggingface/datasets TMPDIR: None tempfile.gettempdir(): /tmp
- Assistant
Thanks — this output explains everything. You are still writing to two places that almost certainly have tight quotas: 🚨 1. /tmp tempfile.gettempdir() = /tmp This is what HF uses for temporary download chunks before moving files into the cache. On most lab systems, /tmp is tiny (like 1–2GB) and quota-restricted. ➡️ This is almost certainly the source of your “Disk quota exceeded” error.
🚨 2. Your home directory cache is still active Even though you set: iniCopy codeHF_HOME=/cs/labs/oabend/shachar.don/hf_cache HF_DATASETS_CACHE=/cs/labs/oabend/shachar.don/hf_cache
The output shows: swiftCopy codeHF default cache dir: /cs/usr/shachar.don/.cache/huggingface Datasets default cache: /cs/usr/shachar.don/.cache/huggingface/datasets
This means: The datasets library is still reading the defaults, because you didn’t set these BEFORE importing datasets. So it’s probably storing metadata/config/partial downloads in your home directory quotas, which are small on cluster systems.
✅ Fix: Set ALL ENV VARS before importing datasets or huggingface_hub Here is the correct block — put it at the very top of your script: pythonCopy codeimport os
redirect all HF cache areas
os.environ["HF_HOME"] = "/cs/scratch/shachar/hf" os.environ["HF_HUB_CACHE"] = "/cs/scratch/shachar/hf/hub" os.environ["HF_DATASETS_CACHE"] = "/cs/scratch/shachar/hf/datasets" os.environ["HF_METRICS_CACHE"] = "/cs/scratch/shachar/hf/metrics"
redirect all temporary file operations away from /tmp
os.environ["TMPDIR"] = "/cs/scratch/shachar/tmp" os.environ["TEMP"] = "/cs/scratch/shachar/tmp" os.environ["TMP"] = "/cs/scratch/shachar/tmp"
os.makedirs("/cs/scratch/shachar/tmp", exist_ok=True)
import AFTER setting env vars
from datasets import load_dataset
🚨 Also clean the old caches in your home dir Run this once: bashCopy coderm -rf /cs/usr/shachar.don/.cache/huggingface rm -rf /cs/usr/shachar.don/.cache/datasets
✔️ After this, rerun your diagnostic prints You should now see: swiftCopy codeHF default cache dir: /cs/scratch/shachar/hf Datasets default cache: /cs/scratch/shachar/hf/datasets tempfile.gettempdir(): /cs/scratch/shachar/tmp
If you want, paste the updated diagnostic output here and I’ll confirm it's fixed.
- User
please change "/cs/scratch/shachar" to "/cs/labs/oabend/shachar.don"
- Assistant
Here is the corrected block with all paths set to /cs/labs/oabend/shachar.don exactly as requested: pythonCopy codeimport os
Redirect all HuggingFace cache directories
os.environ["HF_HOME"] = "/cs/labs/oabend/shachar.don/hf_cache" os.environ["HF_HUB_CACHE"] = "/cs/labs/oabend/shachar.don/hf_cache/hub" os.environ["HF_DATASETS_CACHE"] = "/cs/labs/oabend/shachar.don/hf_cache/datasets" os.environ["HF_METRICS_CACHE"] = "/cs/labs/oabend/shachar.don/hf_cache/metrics"
Redirect temporary files (VERY important to avoid /tmp quota issues)
os.environ["TMPDIR"] = "/cs/labs/oabend/shachar.don/hf_tmp" os.environ["TEMP"] = "/cs/labs/oabend/shachar.don/hf_tmp" os.environ["TMP"] = "/cs/labs/oabend/shachar.don/hf_tmp"
Create dirs if missing
os.makedirs("/cs/labs/oabend/shachar.don/hf_tmp", exist_ok=True) os.makedirs("/cs/labs/oabend/shachar.don/hf_cache", exist_ok=True)
Import AFTER setting env vars
from datasets import load_dataset
Make sure to delete the old home cache once: bashCopy coderm -rf /cs/usr/shachar.don/.cache/huggingface rm -rf /cs/usr/shachar.don/.cache/datasets
Expected correct output after running your diagnostics: swiftCopy codeHF default cache dir: /cs/labs/oabend/shachar.don/hf_cache Datasets default cache: /cs/labs/oabend/shachar.don/hf_cache/datasets tempfile.gettempdir(): /cs/labs/oabend/shachar.don/hf_tmp
If you'd like, paste your new diagnostic output and I can confirm everything is now pointed correctly.
- User
HF_HOME: /cs/labs/oabend/shachar.don/hf_cache HF_DATASETS_CACHE: /cs/labs/oabend/shachar.don/hf_cache HF_METRICS_CACHE: None HF_HUB_CACHE: /cs/labs/oabend/shachar.don/hf_cache HF default cache dir: /cs/usr/shachar.don/.cache/huggingface Datasets default cache: /cs/usr/shachar.don/.cache/huggingface/datasets TMPDIR: None tempfile.gettempdir(): /tmp HF_HOME: /cs/labs/oabend/shachar.don/hf_cache HF_DATASETS_CACHE: /cs/labs/oabend/shachar.don/hf_cache/datasets HF_METRICS_CACHE: /cs/labs/oabend/shachar.don/hf_cache/metrics HF_HUB_CACHE: /cs/labs/oabend/shachar.don/hf_cache/hub HF default cache dir: /cs/usr/shachar.don/.cache/huggingface Datasets default cache: /cs/usr/shachar.don/.cache/huggingface/datasets TMPDIR: /cs/labs/oabend/shachar.don/hf_tmp tempfile.gettempdir(): /tmp
- Assistant
Great — this tells us exactly what’s wrong. You fixed half the problem (the environment variables are set), but datasets is still reading its defaults: swiftCopy codeHF default cache dir: /cs/usr/shachar.don/.cache/huggingface ❌ Datasets default cache: /cs/usr/shachar.don/.cache/huggingface/datasets ❌ tempfile.gettempdir(): /tmp ❌
This means your environment variables were set after datasets or huggingface_hub was imported. 💥 Once those libraries are imported, changing env vars does NOT change their internal cache paths. That’s why your changes didn’t propagate. Let’s fix this cleanly.
✅ Correct fix — ensure env vars are set before any import Your script must look like this: pythonCopy codeimport os
MUST be the first thing in the file:
os.environ["HF_HOME"] = "/cs/labs/oabend/shachar.don/hf_cache" os.environ["HF_HUB_CACHE"] = "/cs/labs/oabend/shachar.don/hf_cache/hub" os.environ["HF_DATASETS_CACHE"] = "/cs/labs/oabend/shachar.don/hf_cache/datasets" os.environ["HF_METRICS_CACHE"] = "/cs/labs/oabend/shachar.don/hf_cache/metrics"
os.environ["TMPDIR"] = "/cs/labs/oabend/shachar.don/hf_tmp" os.environ["TEMP"] = "/cs/labs/oabend/shachar.don/hf_tmp" os.environ["TMP"] = "/cs/labs/oabend/shachar.don/hf_tmp"
os.makedirs("/cs/labs/oabend/shachar.don/hf_tmp", exist_ok=True)
ONLY NOW import datasets or HF!
from datasets import load_dataset import huggingface_hub import tempfile
🔥 You must restart your Python session Because:
datasets.config.HF_DATASETS_CACHE was already initialized in memory
tempfile.gettempdir() is initialized at interpreter startup
So even if you set env vars later, they will not change. ✔️ Restart PyCharm run or ✔️ Restart your shell / job and run again.
🧹 Also delete old conflicting caches Run this once: bashCopy coderm -rf /cs/usr/shachar.don/.cache/huggingface rm -rf /cs/usr/shachar.don/.cache/datasets
🧪 After restart, your diagnostic should show: swiftCopy codeHF default cache dir: /cs/labs/oabend/shachar.don/hf_cache Datasets default cache: /cs/labs/oabend/shachar.don/hf_cache/datasets tempfile.gettempdir(): /cs/labs/oabend/shachar.don/hf_tmp
If you want, send me the updated diagnostics after restart and I’ll verify everything is now correct.
- User
HF_HOME: /cs/labs/oabend/shachar.don/hf_cache HF_DATASETS_CACHE: /cs/labs/oabend/shachar.don/hf_cache HF_METRICS_CACHE: None HF_HUB_CACHE: /cs/labs/oabend/shachar.don/hf_cache HF default cache dir: /cs/usr/shachar.don/.cache/huggingface Datasets default cache: /cs/usr/shachar.don/.cache/huggingface/datasets TMPDIR: None tempfile.gettempdir(): /tmp HF_HOME: /cs/labs/oabend/shachar.don/hf_cache HF_DATASETS_CACHE: /cs/labs/oabend/shachar.don/hf_cache/datasets HF_METRICS_CACHE: /cs/labs/oabend/shachar.don/hf_cache/metrics HF_HUB_CACHE: /cs/labs/oabend/shachar.don/hf_cache/hub HF default cache dir: /cs/usr/shachar.don/.cache/huggingface Datasets default cache: /cs/usr/shachar.don/.cache/huggingface/datasets TMPDIR: /cs/labs/oabend/shachar.don/hf_tmp tempfile.gettempdir(): /tmp
but this dir is still being created during download! /cs/usr/shachar.don/.cache/huggingface/hub/datasets--shachardon--ShareLM
- Assistant
You're extremely close — the only issue left is that the environment variables are still being set after some Hugging Face module is imported, even indirectly. That means something in your code (or PyCharm / an imported module) loads huggingface_hub or datasets before you set the env vars. When that happens, Hugging Face locks in the cache path: swiftCopy codeHF default cache dir: /cs/usr/shachar.don/.cache/huggingface
…and will ignore your environment variables for the rest of the process. That is exactly why this directory is still being created: swiftCopy code/cs/usr/shachar.don/.cache/huggingface/hub/datasets--shachardon--ShareLM
✅ Guaranteed Fix
- Put all environment variables at the VERY TOP of your script, even before any other imports Your script must start like this: pythonCopy codeimport os
MUST BE FIRST — absolutely no other imports before this
os.environ["HF_HOME"] = "/cs/labs/oabend/shachar.don/hf_cache" os.environ["HF_HUB_CACHE"] = "/cs/labs/oabend/shachar.don/hf_cache/hub" os.environ["HF_DATASETS_CACHE"] = "/cs/labs/oabend/shachar.don/hf_cache/datasets" os.environ["HF_METRICS_CACHE"] = "/cs/labs/oabend/shachar.don/hf_cache/metrics"
os.environ["TMPDIR"] = "/cs/labs/oabend/shachar.don/hf_tmp" os.environ["TMP"] = "/cs/labs/oabend/shachar.don/hf_tmp" os.environ["TEMP"] = "/cs/labs/oabend/shachar.don/hf_tmp"
os.makedirs("/cs/labs/oabend/shachar.don/hf_tmp", exist_ok=True)
only now import anything else
from datasets import load_dataset
- Restart your Python interpreter completely (very important) PyCharm keeps the interpreter warm, so previously imported modules stay loaded and ignore your new env vars. In PyCharm:
Click Run → Stop
Click Run → Run ‘script’
Or restart the console
If you're running from a notebook or REPL: restart the kernel. 3. Delete old caches so you can see if they get recreated bashCopy coderm -rf /cs/usr/shachar.don/.cache/huggingface rm -rf /cs/usr/shachar.don/.cache/datasets
- Run your script again
🔍 Why your test prints still show defaults This part of your output: swiftCopy codeHF default cache dir: /cs/usr/shachar.don/.cache/huggingface Datasets default cache: /cs/usr/shachar.don/.cache/huggingface/datasets
proves that datasets was imported before you set: nginxCopy codeHF_HOME HF_DATASETS_CACHE HF_HUB_CACHE
Once HuggingFace imports, it initializes global constants and ignores further env changes.
🔥 Critical detail: ANY imported module may load HF If your script imports anything before setting env vars, it may indirectly import HF: Example: pythonCopy codefrom transformers import AutoTokenizer
or pythonCopy codeimport evaluate
or even something that imports transformers internally. So ensure NO imports above the env-var block.
If you want, paste the first 20 lines of your script I’ll point out the exact import that’s blocking this.
- User
OSError: [Errno 122] Disk quota exceeded: '../../blobs/XXX' -> '/cs/usr/shachar.don/.cache/huggingface/hub/datasets--shachardon--ShareLM/snapshots/XXX/wildchat_formatted_1036000.parquet'
- Assistant
It should not be writing to /cs/usr/shachar.don/.cache/huggingface/... if your environment variables are set correctly — unless something in your environment is still wrong or being overridden. Let's go through the exact remaining causes and how to fix them.
✅ Why is HF still writing to ~/.cache/huggingface? HF uses this priority order:
HF_HOME
HF_HUB_CACHE, HF_DATASETS_CACHE, HF_METRICS_CACHE
XDG_CACHE_HOME
~/.cache/huggingface ← your problematic directory
So HF only touches ~/.cache/huggingface if one of these is missing, or the process is launched with a different environment.
🔥 Cause 1: You were missing HF_DATASETS_CACHE Your old environment showed: rubyCopy codeHF_DATASETS_CACHE: /cs/labs/oabend/shachar.don/hf_cache
But HuggingFace actually expects: swiftCopy codeHF_DATASETS_CACHE: /cs/labs/oabend/shachar.don/hf_cache/datasets
You fixed that later — good.
🔥 Cause 2: You were missing HF_HUB_CACHE Fixed later as well.
❗ Cause 3 (most likely): your Python process is NOT picking up your env vars This is extremely common on CS lab machines. Possibilities: ✔ You set the vars in bash, but launched Python via srun, ssh, conda, or PyCharm → env lost ✔ You set the vars in .bashrc, but the job uses /bin/sh or a non-interactive shell ✔ The notebook kernel started before the variables were set ✔ You printed the variables in bash, but not inside Python 👉 You MUST confirm inside the running Python process.
✅ Run this inside Python: pythonCopy codeimport os for k in ["HF_HOME", "HF_DATASETS_CACHE", "HF_HUB_CACHE", "XDG_CACHE_HOME", "TMPDIR"]: print(k, "=", os.environ.get(k))
You must see: swiftCopy codeHF_HOME = /cs/labs/oabend/shachar.don/hf_cache HF_DATASETS_CACHE = /cs/labs/oabend/shachar.don/hf_cache/datasets HF_HUB_CACHE = /cs/labs/oabend/shachar.don/hf_cache/hub XDG_CACHE_HOME = /cs/labs/oabend/shachar.don/.cache TMPDIR = /cs/labs/oabend/shachar.don/hf_tmp
If ANY of them print None — HF will fall back to your home directory, which is exactly the error you're seeing.
🔥 Cause 4: HF internally caches snapshots in two different places Even if HF_HOME is set, datasets sometimes create auxiliary dirs using:
hf_hub_download() defaults → goes to HF_HUB_CACHE
datasets.load_dataset() may call appdirs.user_cache_dir() → this returns ~/.cache/huggingface
Unless you override XDG_CACHE_HOME, HF datasets still use ~/.cache. 👉 You must set: bashCopy codeexport XDG_CACHE_HOME=/cs/labs/oabend/shachar.don/.cache
Otherwise HF datasets will STILL write into: bashCopy code~/.cache/huggingface/datasets--...
This exactly matches your problem: swiftCopy code/cs/usr/shachar.don/.cache/huggingface/hub/datasets--...
✅ Full final environment (put in .bashrc, .zshrc, or job script) bashCopy codeexport HF_HOME=/cs/labs/oabend/shachar.don/hf_cache export HF_DATASETS_CACHE=/cs/labs/oabend/shachar.don/hf_cache/datasets export HF_HUB_CACHE=/cs/labs/oabend/shachar.don/hf_cache/hub export HF_METRICS_CACHE=/cs/labs/oabend/shachar.don/hf_cache/metrics
critical – prevents fallback to ~/.cache
export XDG_CACHE_HOME=/cs/labs/oabend/shachar.don/.cache
temp dir
export TMPDIR=/cs/labs/oabend/shachar.don/hf_tmp
✔ Finally: delete the unwanted HF cache so HF stops using it But ONLY after the environment is correct: bashCopy coderm -rf /cs/usr/shachar.don/.cache/huggingface
If you delete it before fixing the env, HF will just recreate it again.
🧪 If you want, paste me the output of os.environ inside Python and I’ll tell you the exact missing variable.