1import os
2import shutil
3import urllib.request
4from datetime import datetime, timedelta
5
6BASE_URL = "https://download.gatedata.org"
7HOURLY_TYPES = {"orderbooks", "orderbooks_slice"}
8
9
10def build_url(biz: str, type_: str, market: str, dt_utc: datetime) -> str:
11 ym = dt_utc.strftime("%Y%m") # folder: 202304
12 ymdh = dt_utc.strftime("%Y%m%d%H") # hourly: 2023042505
13
14 if type_ == "orderbooks_slice":
15 fname = f"{market}-{ymdh}.gz"
16 elif type_ == "orderbooks":
17 fname = f"{market}-{ymdh}.csv.gz"
18 else:
19 fname = f"{market}-{ym}.csv.gz"
20
21 return f"{BASE_URL}/{biz}/{type_}/{ym}/{fname}"
22
23
24def download_and_extract(url: str, save_dir: str) -> str:
25 # build sub-folder from URL: {save_dir}/{biz}/{type}
26 parts = url.split("/")
27 save_dir = os.path.join(save_dir, parts[3], parts[4])
28 os.makedirs(save_dir, exist_ok=True)
29 gz_name = parts[-1]
30 gz_path = os.path.join(save_dir, gz_name)
31
32 print(f"↓ {url}")
33 urllib.request.urlretrieve(url, gz_path)
34
35 # orderbooks_slice → .txt, others (.csv.gz) → .csv
36 if gz_name.endswith(".csv.gz"):
37 out_path = gz_path[: -len(".csv.gz")] + ".csv"
38 else:
39 out_path = gz_path[: -len(".gz")] + ".txt"
40
41 with gzip.open(gz_path, "rb") as f_in, open(out_path, "wb") as f_out:
42 shutil.copyfileobj(f_in, f_out)
43
44 os.remove(gz_path)
45 print(f" saved → {out_path}")
46 return out_path
47
48
49def batch_download(biz: str, type_: str, market: str,
50 save_dir: str,
51 start_dt: datetime,
52 end_dt: datetime = None):
53 """
54 If end_dt is omitted, downloads only the single file for start_dt.
55
56 Monthly granularity types: iterates month by month.
57 e.g. start_dt=datetime(2026,1,1), end_dt=datetime(2026,3,1)
58 → downloads 202601 / 202602 / 202603
59
60 Hourly granularity types (orderbooks / orderbooks_slice): iterates hour by hour.
61 e.g. start_dt=datetime(2026,1,1,0), end_dt=datetime(2026,1,2,23)
62 → downloads 2026010100 ~ 2026010223, 48 files in total
63 """
64 if end_dt is None:
65 end_dt = start_dt
66
67 if type_ in HOURLY_TYPES:
68 dt = start_dt
69 while dt <= end_dt:
70 download_and_extract(build_url(biz, type_, market, dt), save_dir)
71 dt += timedelta(hours=1)
72 else:
73 dt = start_dt.replace(day=1, hour=0, minute=0, second=0)
74 end = end_dt.replace(day=1, hour=0, minute=0, second=0)
75 while dt <= end:
76 download_and_extract(build_url(biz, type_, market, dt), save_dir)
77 dt = (dt.replace(day=1) + timedelta(days=32)).replace(day=1)
78
79
80if __name__ == "__main__":
81 # ── Spot ──────────────────────────────────────────────────────────────
82 # Single download
83 batch_download("spot", "deals", "BTC_USDT", "./data", datetime(2023, 4, 25))
84 # batch_download("spot", "candlesticks_1m", "BTC_USDT", "./data", datetime(2023, 4, 25))
85 # batch_download("spot", "candlesticks_5m", "BTC_USDT", "./data", datetime(2023, 4, 25))
86 # batch_download("spot", "candlesticks_1h", "BTC_USDT", "./data", datetime(2023, 4, 25))
87 # batch_download("spot", "candlesticks_4h", "BTC_USDT", "./data", datetime(2023, 4, 25))
88 # batch_download("spot", "candlesticks_1d", "BTC_USDT", "./data", datetime(2023, 4, 25))
89 # batch_download("spot", "candlesticks_7d", "BTC_USDT", "./data", datetime(2023, 4, 25))
90 # batch_download("spot", "orderbooks", "BTC_USDT", "./data", datetime(2023, 4, 25, 5))
91 # batch_download("spot", "orderbooks_slice", "BTC_USDT", "./data", datetime(2023, 4, 25, 5))
92 # Batch download
93 batch_download("spot", "deals", "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
94 # batch_download("spot", "candlesticks_1m", "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
95 # batch_download("spot", "candlesticks_5m", "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
96 # batch_download("spot", "candlesticks_1h", "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
97 # batch_download("spot", "candlesticks_4h", "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
98 # batch_download("spot", "candlesticks_1d", "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
99 # batch_download("spot", "candlesticks_7d", "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
100 # batch_download("spot", "orderbooks", "BTC_USDT", "./data", datetime(2023, 4, 25, 0), datetime(2023, 4, 25, 23))
101 # batch_download("spot", "orderbooks_slice", "BTC_USDT", "./data", datetime(2023, 4, 25, 0), datetime(2023, 4, 25, 23))
102
103 # ── futures_usdt ──────────────────────────────────────────────────────────
104 # Single download
105 # batch_download("futures_usdt", "trades", "BTC_USDT", "./data", datetime(2023, 4, 25))
106 # batch_download("futures_usdt", "candlesticks_10s", "BTC_USDT", "./data", datetime(2023, 4, 25))
107 # batch_download("futures_usdt", "candlesticks_1m", "BTC_USDT", "./data", datetime(2023, 4, 25))
108 # batch_download("futures_usdt", "candlesticks_1h", "BTC_USDT", "./data", datetime(2023, 4, 25))
109 # batch_download("futures_usdt", "candlesticks_4h", "BTC_USDT", "./data", datetime(2023, 4, 25))
110 # batch_download("futures_usdt", "candlesticks_1d", "BTC_USDT", "./data", datetime(2023, 4, 25))
111 # batch_download("futures_usdt", "candlesticks_7d", "BTC_USDT", "./data", datetime(2023, 4, 25))
112 # batch_download("futures_usdt", "orderbooks", "BTC_USDT", "./data", datetime(2023, 4, 25, 5))
113 # batch_download("futures_usdt", "orderbooks_slice", "BTC_USDT", "./data", datetime(2023, 4, 25, 5))
114 # batch_download("futures_usdt", "mark_prices", "BTC_USDT", "./data", datetime(2023, 4, 25))
115 # batch_download("futures_usdt", "funding_applies", "BTC_USDT", "./data", datetime(2023, 4, 25))
116 # batch_download("futures_usdt", "funding_updates", "BTC_USDT", "./data", datetime(2023, 4, 25))
117 # Batch download
118 # batch_download("futures_usdt", "trades", "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
119 # batch_download("futures_usdt", "candlesticks_10s", "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
120 # batch_download("futures_usdt", "candlesticks_1m", "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
121 # batch_download("futures_usdt", "candlesticks_1h", "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
122 # batch_download("futures_usdt", "candlesticks_4h", "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
123 # batch_download("futures_usdt", "candlesticks_1d", "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
124 # batch_download("futures_usdt", "candlesticks_7d", "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
125 # batch_download("futures_usdt", "orderbooks", "BTC_USDT", "./data", datetime(2023, 4, 25, 0), datetime(2023, 4, 25, 23))
126 # batch_download("futures_usdt", "orderbooks_slice", "BTC_USDT", "./data", datetime(2023, 4, 25, 0), datetime(2023, 4, 25, 23))
127 # batch_download("futures_usdt", "mark_prices", "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
128 # batch_download("futures_usdt", "funding_applies", "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
129 # batch_download("futures_usdt", "funding_updates", "BTC_USDT", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
130
131 # ── futures_btc ────────────────────────────────────────────────────────
132 # Single download
133 # batch_download("futures_btc", "trades", "BTC_USD", "./data", datetime(2023, 4, 25))
134 # batch_download("futures_btc", "candlesticks_10s", "BTC_USD", "./data", datetime(2023, 4, 25))
135 # batch_download("futures_btc", "candlesticks_1m", "BTC_USD", "./data", datetime(2023, 4, 25))
136 # batch_download("futures_btc", "candlesticks_1h", "BTC_USD", "./data", datetime(2023, 4, 25))
137 # batch_download("futures_btc", "candlesticks_4h", "BTC_USD", "./data", datetime(2023, 4, 25))
138 # batch_download("futures_btc", "candlesticks_1d", "BTC_USD", "./data", datetime(2023, 4, 25))
139 # batch_download("futures_btc", "candlesticks_7d", "BTC_USD", "./data", datetime(2023, 4, 25))
140 # batch_download("futures_btc", "orderbooks", "BTC_USD", "./data", datetime(2023, 4, 25, 5))
141 # batch_download("futures_btc", "orderbooks_slice", "BTC_USD", "./data", datetime(2023, 4, 25, 5))
142 # batch_download("futures_btc", "mark_prices", "BTC_USD", "./data", datetime(2023, 4, 25))
143 # batch_download("futures_btc", "funding_applies", "BTC_USD", "./data", datetime(2023, 4, 25))
144 # batch_download("futures_btc", "funding_updates", "BTC_USD", "./data", datetime(2023, 4, 25))
145 # Batch download
146 # batch_download("futures_btc", "trades", "BTC_USD", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
147 # batch_download("futures_btc", "candlesticks_10s", "BTC_USD", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
148 # batch_download("futures_btc", "candlesticks_1m", "BTC_USD", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
149 # batch_download("futures_btc", "candlesticks_1h", "BTC_USD", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
150 # batch_download("futures_btc", "candlesticks_4h", "BTC_USD", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
151 # batch_download("futures_btc", "candlesticks_1d", "BTC_USD", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
152 # batch_download("futures_btc", "candlesticks_7d", "BTC_USD", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
153 # batch_download("futures_btc", "orderbooks", "BTC_USD", "./data", datetime(2023, 4, 25, 0), datetime(2023, 4, 25, 23))
154 # batch_download("futures_btc", "orderbooks_slice", "BTC_USD", "./data", datetime(2023, 4, 25, 0), datetime(2023, 4, 25, 23))
155 # batch_download("futures_btc", "mark_prices", "BTC_USD", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
156 # batch_download("futures_btc", "funding_applies", "BTC_USD", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
157 # batch_download("futures_btc", "funding_updates", "BTC_USD", "./data", datetime(2023, 1, 1), datetime(2023, 4, 1))
