fetch.c 9.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412
  1. /* vim: tabstop=4 shiftwidth=4 noexpandtab
  2. * This file is part of ToaruOS and is released under the terms
  3. * of the NCSA / University of Illinois License - see LICENSE.md
  4. * Copyright (C) 2015-2018 K. Lange
  5. *
  6. * fetch - Retreive documents from HTTP servers.
  7. *
  8. */
  9. #include <stdio.h>
  10. #include <string.h>
  11. #include <stdlib.h>
  12. #include <getopt.h>
  13. #include <time.h>
  14. #include <sys/time.h>
  15. #include <termios.h>
  16. #include <unistd.h>
  17. #include <toaru/hashmap.h>
  18. #define SIZE 512
  19. #define BOUNDARY "------ToaruOSFetchUploadBoundary"
  20. struct http_req {
  21. char domain[SIZE];
  22. char path[SIZE];
  23. };
  24. struct {
  25. int show_headers;
  26. const char * output_file;
  27. const char * cookie;
  28. FILE * out;
  29. int prompt_password;
  30. const char * upload_file;
  31. char * password;
  32. int show_progress;
  33. size_t content_length;
  34. size_t size;
  35. struct timeval start;
  36. int calculate_output;
  37. int slow_upload;
  38. int machine_readable;
  39. } fetch_options = {0};
  40. void parse_url(char * d, struct http_req * r) {
  41. if (strstr(d, "http://") == d) {
  42. d += strlen("http://");
  43. char * s = strstr(d, "/");
  44. if (!s) {
  45. strcpy(r->domain, d);
  46. strcpy(r->path, "");
  47. } else {
  48. *s = 0;
  49. s++;
  50. strcpy(r->domain, d);
  51. strcpy(r->path, s);
  52. }
  53. } else {
  54. fprintf(stderr, "sorry, can't parse %s\n", d);
  55. exit(1);
  56. }
  57. }
  58. #define BAR_WIDTH 20
  59. #define bar_perc "||||||||||||||||||||"
  60. #define bar_spac " "
  61. void print_progress(int force) {
  62. static uint64_t last_size = 0;
  63. if (!fetch_options.show_progress) return;
  64. if (!force && (last_size + 102400 > fetch_options.size)) return;
  65. last_size = fetch_options.size;
  66. struct timeval now;
  67. gettimeofday(&now, NULL);
  68. fprintf(stderr,"\033[?25l\033[G%6dkB",(int)fetch_options.size/1024);
  69. if (fetch_options.content_length) {
  70. int percent = (fetch_options.size * BAR_WIDTH) / (fetch_options.content_length);
  71. fprintf(stderr," / %6dkB [%.*s%.*s]", (int)fetch_options.content_length/1024, percent,bar_perc,BAR_WIDTH-percent,bar_spac);
  72. }
  73. double timediff = (double)(now.tv_sec - fetch_options.start.tv_sec) + (double)(now.tv_usec - fetch_options.start.tv_usec)/1000000.0;
  74. if (timediff > 0.0) {
  75. double rate = (double)(fetch_options.size) / timediff;
  76. double s = rate/(1024.0) * 8.0;
  77. if (s > 1024.0) {
  78. fprintf(stderr," %.2f Mbps", s/1024.0);
  79. } else {
  80. fprintf(stderr," %.2f Kbps", s);
  81. }
  82. if (fetch_options.content_length) {
  83. if (rate > 0.0) {
  84. double remaining = (double)(fetch_options.content_length - fetch_options.size) / rate;
  85. fprintf(stderr," (%.2f sec remaining)", remaining);
  86. }
  87. }
  88. }
  89. fprintf(stderr,"\033[K\033[?25h");
  90. fflush(stderr);
  91. }
  92. int usage(char * argv[]) {
  93. fprintf(stderr,
  94. "fetch - download files over HTTP\n"
  95. "\n"
  96. "usage: %s [-hOvmp?] [-c cookie] [-o file] [-u file] [-s speed] URL\n"
  97. "\n"
  98. " -h \033[3mshow headers\033[0m\n"
  99. " -O \033[3msave the file based on the filename in the URL\033[0m\n"
  100. " -v \033[3mshow progress\033[0m\n"
  101. " -m \033[3mmachine readable output\033[0m\n"
  102. " -p \033[3mprompt for password\033[0m\n"
  103. " -c ... \033[3mset cookies\033[0m\n"
  104. " -o ... \033[3msave to the specified file\033[0m\n"
  105. " -u ... \033[3mupload the specified file\033[0m\n"
  106. " -s ... \033[3mspecify the speed for uploading slowly\033[0m\n"
  107. " -? \033[3mshow this help text\033[0m\n"
  108. "\n", argv[0]);
  109. return 1;
  110. }
  111. int collect_password(char * password) {
  112. fprintf(stdout, "Password for upload: ");
  113. fflush(stdout);
  114. /* Disable echo */
  115. struct termios old, new;
  116. tcgetattr(fileno(stdin), &old);
  117. new = old;
  118. new.c_lflag &= (~ECHO);
  119. tcsetattr(fileno(stdin), TCSAFLUSH, &new);
  120. fgets(password, 1024, stdin);
  121. password[strlen(password)-1] = '\0';
  122. tcsetattr(fileno(stdin), TCSAFLUSH, &old);
  123. fprintf(stdout, "\n");
  124. return 0;
  125. }
  126. void read_http_line(char * buf, FILE * f) {
  127. memset(buf, 0x00, 256);
  128. fgets(buf, 255, f);
  129. char * _r = strchr(buf, '\r');
  130. if (_r) {
  131. *_r = '\0';
  132. }
  133. if (!_r) {
  134. _r = strchr(buf, '\n'); /* that's not right, but, whatever */
  135. if (_r) {
  136. *_r = '\0';
  137. }
  138. }
  139. }
  140. void bad_response(void) {
  141. fprintf(stderr, "Bad response.\n");
  142. exit(1);
  143. }
  144. int http_fetch(FILE * f) {
  145. hashmap_t * headers = hashmap_create(10);
  146. /* Parse response */
  147. {
  148. char buf[256];
  149. read_http_line(buf, f);
  150. char * elements[3];
  151. elements[0] = buf;
  152. elements[1] = strchr(elements[0], ' ');
  153. if (!elements[1]) bad_response();
  154. *elements[1] = '\0';
  155. elements[1]++;
  156. elements[2] = strchr(elements[1], ' ');
  157. if (!elements[2]) bad_response();
  158. *elements[2] = '\0';
  159. elements[2]++;
  160. if (strcmp(elements[1], "200")) {
  161. fprintf(stderr, "Bad response code: %s\n", elements[1]);
  162. return 1;
  163. }
  164. }
  165. /* Parse headers */
  166. while (1) {
  167. char buf[256];
  168. read_http_line(buf, f);
  169. if (!*buf) {
  170. break;
  171. }
  172. /* Split */
  173. char * name = buf;
  174. char * value = strstr(buf, ": ");
  175. if (!value) bad_response();
  176. *value = '\0';
  177. value += 2;
  178. hashmap_set(headers, name, strdup(value));
  179. }
  180. if (fetch_options.show_headers) {
  181. list_t * hash_keys = hashmap_keys(headers);
  182. foreach(_key, hash_keys) {
  183. char * key = (char *)_key->value;
  184. fprintf(stderr, "[%s] = %s\n", key, (char*)hashmap_get(headers, key));
  185. }
  186. list_free(hash_keys);
  187. free(hash_keys);
  188. }
  189. /* determine how many bytes we should read now */
  190. if (!hashmap_has(headers, "Content-Length")) {
  191. fprintf(stderr, "Don't know how much to read.\n");
  192. return 1;
  193. }
  194. int bytes_to_read = atoi(hashmap_get(headers, "Content-Length"));
  195. fetch_options.content_length = bytes_to_read;
  196. gettimeofday(&fetch_options.start, NULL);
  197. while (bytes_to_read > 0) {
  198. char buf[1024];
  199. size_t r = fread(buf, 1, bytes_to_read < 1024 ? bytes_to_read : 1024, f);
  200. fwrite(buf, 1, r, fetch_options.out);
  201. fetch_options.size += r;
  202. print_progress(0);
  203. if (fetch_options.machine_readable && fetch_options.content_length) {
  204. fprintf(stdout,"%d %d\n",(int)fetch_options.size, (int)fetch_options.content_length);
  205. }
  206. bytes_to_read -= r;
  207. }
  208. print_progress(1);
  209. return 0;
  210. }
  211. int main(int argc, char * argv[]) {
  212. int opt;
  213. while ((opt = getopt(argc, argv, "?c:hmo:Opu:vs:")) != -1) {
  214. switch (opt) {
  215. case '?':
  216. return usage(argv);
  217. case 'O':
  218. fetch_options.calculate_output = 1;
  219. break;
  220. case 'c':
  221. fetch_options.cookie = optarg;
  222. break;
  223. case 'h':
  224. fetch_options.show_headers = 1;
  225. break;
  226. case 'o':
  227. fetch_options.output_file = optarg;
  228. break;
  229. case 'u':
  230. fetch_options.upload_file = optarg;
  231. break;
  232. case 'v':
  233. fetch_options.show_progress = 1;
  234. break;
  235. case 'm':
  236. fetch_options.machine_readable = 1;
  237. break;
  238. case 'p':
  239. fetch_options.prompt_password = 1;
  240. break;
  241. case 's':
  242. fetch_options.slow_upload = atoi(optarg);
  243. break;
  244. }
  245. }
  246. if (optind >= argc) {
  247. return usage(argv);
  248. }
  249. struct http_req my_req;
  250. parse_url(argv[optind], &my_req);
  251. char file[100];
  252. sprintf(file, "/dev/net/%s", my_req.domain);
  253. if (fetch_options.calculate_output) {
  254. char * tmp = strdup(my_req.path);
  255. char * x = strrchr(tmp,'/');
  256. if (x) {
  257. tmp = x + 1;
  258. }
  259. fetch_options.output_file = tmp;
  260. }
  261. fetch_options.out = stdout;
  262. if (fetch_options.output_file) {
  263. fetch_options.out = fopen(fetch_options.output_file, "w+");
  264. }
  265. FILE * f = fopen(file,"r+");
  266. if (!f) {
  267. fprintf(stderr, "Nope.\n");
  268. return 1;
  269. }
  270. if (fetch_options.prompt_password) {
  271. fetch_options.password = malloc(100);
  272. collect_password(fetch_options.password);
  273. }
  274. if (fetch_options.upload_file) {
  275. FILE * in_file = fopen(fetch_options.upload_file, "r");
  276. srand(time(NULL));
  277. int boundary_fuzz = rand();
  278. char tmp[512];
  279. size_t out_size = 0;
  280. if (fetch_options.password) {
  281. out_size += sprintf(tmp,
  282. "--" BOUNDARY "%08x\r\n"
  283. "Content-Disposition: form-data; name=\"password\"\r\n"
  284. "\r\n"
  285. "%s\r\n",boundary_fuzz, fetch_options.password);
  286. }
  287. out_size += strlen("--" BOUNDARY "00000000\r\n"
  288. "Content-Disposition: form-data; name=\"file\"; filename=\"\"\r\n"
  289. "Content-Type: application/octet-stream\r\n"
  290. "\r\n"
  291. /* Data goes here */
  292. "\r\n"
  293. "--" BOUNDARY "00000000" "--\r\n");
  294. out_size += strlen(fetch_options.upload_file);
  295. fseek(in_file, 0, SEEK_END);
  296. out_size += ftell(in_file);
  297. fseek(in_file, 0, SEEK_SET);
  298. fprintf(f,
  299. "POST /%s HTTP/1.0\r\n"
  300. "User-Agent: curl/7.35.0\r\n"
  301. "Host: %s\r\n"
  302. "Accept: */*\r\n"
  303. "Content-Length: %d\r\n"
  304. "Content-Type: multipart/form-data; boundary=" BOUNDARY "%08x\r\n"
  305. "\r\n", my_req.path, my_req.domain, (int)out_size, boundary_fuzz);
  306. fprintf(f,"%s",tmp);
  307. fprintf(f,
  308. "--" BOUNDARY "%08x\r\n"
  309. "Content-Disposition: form-data; name=\"file\"; filename=\"%s\"\r\n"
  310. "Content-Type: application/octet-stream\r\n"
  311. "\r\n", boundary_fuzz, fetch_options.upload_file);
  312. while (!feof(in_file)) {
  313. char buf[1024];
  314. size_t r = fread(buf, 1, 1024, in_file);
  315. fwrite(buf, 1, r, f);
  316. if (fetch_options.slow_upload) {
  317. usleep(1000 * fetch_options.slow_upload); /* TODO fix terrible network stack; hopefully this ensures we send stuff right. */
  318. }
  319. }
  320. fclose(in_file);
  321. fprintf(f,"\r\n--" BOUNDARY "%08x--\r\n", boundary_fuzz);
  322. fflush(f);
  323. } else if (fetch_options.cookie) {
  324. fprintf(f,
  325. "GET /%s HTTP/1.0\r\n"
  326. "User-Agent: curl/7.35.0\r\n"
  327. "Host: %s\r\n"
  328. "Accept: */*\r\n"
  329. "Cookie: %s\r\n"
  330. "\r\n", my_req.path, my_req.domain, fetch_options.cookie);
  331. } else {
  332. fprintf(f,
  333. "GET /%s HTTP/1.0\r\n"
  334. "User-Agent: curl/7.35.0\r\n"
  335. "Host: %s\r\n"
  336. "Accept: */*\r\n"
  337. "\r\n", my_req.path, my_req.domain);
  338. }
  339. http_fetch(f);
  340. fflush(fetch_options.out);
  341. if (fetch_options.show_progress) {
  342. fprintf(stderr,"\n");
  343. }
  344. if (fetch_options.machine_readable) {
  345. fprintf(stdout,"done\n");
  346. }
  347. return 0;
  348. }