Main Page | Modules | File List | Globals

main.c

00001 /*
00002  * Copyright (c) 2005, 2006 by KoanLogic s.r.l. <http://www.koanlogic.com>
00003  * All rights reserved.
00004  *
00005  * This file is part of KLone, and as such it is subject to the license stated
00006  * in the LICENSE file which you have received as part of this distribution.
00007  *
00008  * $Id: main.c,v 1.43 2007/12/23 10:28:45 tat Exp $
00009  */
00010 
00011 #include "klone_conf.h"
00012 #include <sys/stat.h>
00013 #ifdef HAVE_SYS_DIR
00014 #include <sys/dir.h>
00015 #endif
00016 #include <sys/types.h>
00017 #include <dirent.h>
00018 #include <stdlib.h>
00019 #include <ctype.h>
00020 #include <stdio.h>
00021 #include <fcntl.h>
00022 #ifdef HAVE_UNISTD
00023 #include <unistd.h>
00024 #endif
00025 #ifdef HAVE_GETOPT
00026 #include <getopt.h>
00027 #endif
00028 #include <u/libu.h>
00029 #include <klone/os.h>
00030 #include <klone/request.h>
00031 #include <klone/response.h>
00032 #include <klone/translat.h>
00033 #include <klone/utils.h>
00034 #include <klone/run.h>
00035 #include <klone/mime_map.h>
00036 #include <klone/version.h>
00037 #include "pm.h"
00038 
00039 int facility = LOG_LOCAL0;
00040 
00041 /* command list enums */
00042 enum command_e { CMD_UNKNOWN, CMD_TRANS, CMD_IMPORT };
00043 
00044 /* runtime flags */
00045 enum flags_e { FLAG_NONE, FLAG_VERBOSE };
00046 
00047 typedef struct 
00048 {
00049     char *file_in, *file_out;   /* [trans] input, output file       */
00050     char *uri;                  /* [trans] translated file uri      */
00051     int verbose;                /* >0 when verbose mode is on       */
00052     char **arg;                 /* argv                             */
00053     size_t narg;                /* argc                             */
00054     int cmd;                    /* command to run                   */
00055     char *base_uri;             /* site base uri                    */
00056     int encrypt;                /* >0 when encryption is enabled    */
00057     int compress;               /* >0 when compress is enabled      */
00058     pm_t *comp_patt;            /* compress file pattern            */
00059     pm_t *enc_patt;             /* encrypt file pattern             */
00060     pm_t *excl_patt;            /* exclude file pattern             */
00061     char *key_file;             /* encryption key file name         */
00062     io_t *iom, *iod, *ior;      /* io makefile, io deps             */
00063     size_t ndir, nfile;         /* dir and file count               */
00064     size_t nexcl;               /* # of excluded files (-x)         */
00065 } context_t;
00066 
00067 context_t *ctx;
00068 
00069 #define KL1_FILE_FMT "pg_%s.%s"
00070 
00071 static void usage(void)
00072 {
00073     static const char * us = 
00074 "Usage: klone [-hvV] -c COMMAND OPTIONS ARGUMENTS\n"
00075 "Version: %s - Copyright (c) 2005, 2006, 2007 KoanLogic s.r.l.\n"
00076 "All rights reserved.\n"
00077 "\n"
00078 "       -h            display this help\n"
00079 "       -v            verbose mode\n"
00080 "       -V            print KLone version and exit\n"
00081 "       -c command    command to execute (see COMMAND LIST)\n"
00082 "\n"
00083 "\n"
00084 "    COMMAND LIST:\n"
00085 "       import        import a directory tree in the embedded filesystem\n"
00086 "       translate     convert a file or a dynamic web page to a C file\n"
00087 "\n"
00088 "\n"
00089 "    COMMANDS SYNTAX:\n"
00090 "\n"
00091 "       import OPTIONS dir\n"
00092 "         -b URI      base URI\n"
00093 "         -x pattern  exclude all files whose URI match the given pattern (*)\n"
00094 #ifdef HAVE_LIBOPENSSL
00095 "         -e pattern  encrypt all files whose URI match the given pattern (*)\n"
00096 "         -k key_file encryption key filename\n"
00097 #endif
00098 #ifdef HAVE_LIBZ
00099 "         -z          compress all compressable content (based on MIME types)\n"
00100 "         -Z pattern  compress all files whose URI match the given pattern (*)\n"
00101 #endif
00102 "         dir         directory tree path\n"
00103 "\n"
00104 "         (*) may be used more then once\n"
00105 "\n"
00106 "       translate OPTIONS\n"
00107 #ifdef HAVE_LIBOPENSSL
00108 "         -E          encrypt file content\n"
00109 #endif
00110 "         -i file     input file\n"
00111 #ifdef HAVE_LIBOPENSSL
00112 "         -k key_file encryption key filename\n"
00113 #endif
00114 "         -o file     output file\n"
00115 "         -u URI      URI of translated page\n"
00116 "                     (KLONE_CIPHER_KEY environ var is used if not provided)\n"
00117 #ifdef HAVE_LIBZ
00118 "         -z          compress file content\n"
00119 #endif
00120 "\n";
00121 
00122     fprintf(stderr, us, klone_version());
00123 
00124     exit(EXIT_FAILURE);
00125 }
00126 
00127 static void remove_trailing_slash(char *s)
00128 {
00129     size_t len;
00130     
00131     dbg_ifb (s == NULL) return;
00132     
00133     len = strlen(s);
00134     if(len && s[len - 1] == '/')
00135         s[len - 1] = 0;
00136 }
00137 
00138 static int parse_opt(int argc, char **argv)
00139 {
00140     int ret;
00141     char opts[512];
00142 
00143     if(argc == 1)
00144         usage();
00145 
00146     /* common switches */
00147     strcpy(opts, "hvVx:b:i:o:u:c:");
00148 
00149     /* encryption switches */
00150 #ifdef HAVE_LIBOPENSSL
00151     strcat(opts, "k:e:E");
00152 #endif
00153 
00154     /* compression switches */
00155 #ifdef HAVE_LIBZ
00156     strcat(opts, "zZ:");
00157 #endif
00158 
00159     while((ret = getopt(argc, argv, opts)) != -1)
00160     {
00161         switch(ret)
00162         {
00163         case 'v': /* verbose on */
00164             ctx->verbose++;
00165             break;
00166         case 'V': /* print name/version info and exit */
00167             u_print_version_and_exit();
00168             break;
00169 
00170 #ifdef HAVE_LIBOPENSSL
00171         case 'E': /* encryption on */
00172             ctx->encrypt = 1;
00173             break;
00174         case 'e': /* encrypt file pattern */
00175             dbg_err_if(pm_add(ctx->enc_patt, u_strdup(optarg)));
00176             break;
00177         case 'k': /* encryption key filename */
00178             ctx->key_file = u_strdup(optarg);
00179             warn_err_if(ctx->key_file == NULL);
00180             break;
00181 #endif
00182 
00183 #ifdef HAVE_LIBZ
00184         case 'Z': /* compress file pattern */
00185             ctx->compress = 1;
00186             dbg_err_if(pm_add(ctx->comp_patt, u_strdup(optarg)));
00187             break;
00188         case 'z': /* compress */
00189             ctx->compress = 1;
00190             break;
00191 #endif
00192         case 'c': /* command */
00193             if(!strcasecmp(optarg, "import"))
00194                 ctx->cmd = CMD_IMPORT;
00195             else if(!strcasecmp(optarg, "translate"))
00196                 ctx->cmd = CMD_TRANS;
00197             else
00198                 con_err("unknown command: %s", optarg);
00199             break;
00200         case 'i': /* input file */
00201             ctx->file_in = u_strdup(optarg);
00202             warn_err_if(ctx->file_in == NULL);
00203             break;
00204         case 'x': /* exclude pattern */
00205             dbg_err_if(pm_add(ctx->excl_patt, u_strdup(optarg)));
00206             break;
00207         case 'b': /* base_uri */
00208             ctx->base_uri = u_strdup(optarg);
00209             warn_err_if(ctx->base_uri == NULL);
00210 
00211             if(ctx->base_uri[0] != '/')
00212                 klone_die("base URI must be absolute "
00213                           "(i.e. must start with a '/')");
00214 
00215             remove_trailing_slash(ctx->base_uri);
00216 
00217             break;
00218         case 'o': /* output file */
00219             ctx->file_out = u_strdup(optarg);
00220             warn_err_if(ctx->file_out == NULL);
00221             break;
00222         case 'u': /* translated page uri */
00223             /* skip the first char to avoid MSYS path translation bug
00224              * (see klone-site.c) */
00225             ctx->uri = u_strdup(1+optarg);
00226             warn_err_if(ctx->uri == NULL);
00227 
00228             if(ctx->uri[0] != '/')
00229                 klone_die("URI must be absolute (i.e. must start with a '/')");
00230 
00231             remove_trailing_slash(ctx->uri);
00232 
00233             break;
00234         default:
00235         case 'h': 
00236             usage();
00237         }
00238     }
00239 
00240     klone_die_if(ctx->cmd == 0, "missing command argument (-c)");
00241     ctx->narg = argc - optind;  /* # of args left */
00242     ctx->arg = argv + optind;   
00243 
00244     return 0;
00245 err:
00246     return ~0;
00247 }
00248 
00249 static int set_key_from_file(trans_info_t *pti, const char *key_file)
00250 {
00251     io_t *io = NULL;
00252 
00253     dbg_err_if (pti == NULL);
00254     dbg_err_if (key_file == NULL);
00255     
00256     dbg_err_if(u_file_open(key_file, O_RDONLY, &io));
00257 
00258     dbg_err_if(io_read(io, pti->key, CODEC_CIPHER_KEY_SIZE) <= 0);
00259     
00260     io_free(io);
00261 
00262     return 0;
00263 err:
00264     return ~0;
00265 }
00266 
00267 static int command_trans(void)
00268 {
00269     trans_info_t ti;
00270     const mime_map_t *mm;
00271     struct stat st;
00272     char *key_env;
00273     int key_found = 0;
00274 
00275     if(ctx->narg != 0)
00276         usage();    /* no argument allowed */
00277 
00278     memset(&ti, 0, sizeof(trans_info_t));
00279 
00280     klone_die_if(!ctx->file_in, "input file name required (-i file)");
00281     klone_die_if(!ctx->file_out, "output file name required (-o file)");
00282     klone_die_if(!ctx->uri, "translated page URI required (-u uri)");
00283 
00284     if(ctx->verbose)
00285         con("translating %s to %s (uri: %s)", ctx->file_in, ctx->file_out, 
00286             ctx->uri);
00287 
00288     /* input file */
00289     strncpy(ti.file_in, ctx->file_in, U_FILENAME_MAX);
00290 
00291     /* output file */
00292     strncpy(ti.file_out, ctx->file_out, U_FILENAME_MAX);
00293 
00294     /* uri */
00295     strncpy(ti.uri, ctx->uri, URI_BUFSZ);
00296 
00297     /* zero out the key (some byte could not be overwritten with small keys) */
00298     memset(ti.key, 0, CODEC_CIPHER_KEY_SIZE);
00299 
00300     /* sanity checks */
00301     con_err_ifm(ctx->key_file && !ctx->encrypt, "-k used but -E is missing");
00302 
00303     /* encryption key */
00304     key_env = getenv("KLONE_CIPHER_KEY");
00305     if(key_env && strlen(key_env))
00306     {
00307         key_found = 1;
00308         strncpy(ti.key, key_env, CODEC_CIPHER_KEY_SIZE);
00309     }
00310 
00311     /* if -k has been used the overwrite KLONE_CIPHER_KEY env var (if present)*/
00312     if(ctx->key_file)
00313     {
00314         key_found = 1;
00315         con_err_ifm(set_key_from_file(&ti, ctx->key_file), 
00316             "error reading key file [%s]", ctx->key_file);
00317     }
00318 
00319     if(ctx->encrypt)
00320     {
00321         if(!key_found)
00322             con_err("encryption key required (use -k or KLONE_CIPHER_KEY "
00323                     "environ variable)");
00324         ti.encrypt = 1;
00325     }
00326 
00327     /* set MIME type */
00328     if((mm = u_get_mime_map(ctx->file_in)) != NULL)
00329         strncpy(ti.mime_type, mm->mime_type, MIME_BUFSZ);
00330     else
00331         strncpy(ti.mime_type, "application/octet-stream", MIME_BUFSZ);
00332 
00333     /* compress if requested and the file is compressable (by MIME type) */
00334     if(ctx->compress)
00335         ti.comp = 1;
00336 
00337     /* be sure that the input file exists */
00338     klone_die_if(stat(ctx->file_in, &st), "input file not found");
00339 
00340     ti.file_size = st.st_size;
00341     ti.mtime = st.st_mtime; 
00342 
00343     /* translate it */
00344     dbg_err_if(translate(&ti));
00345 
00346     return 0;
00347 err:
00348     /* delete output file on error */
00349     u_remove(ti.file_out);
00350     con(" ");
00351     return ~0;
00352 }
00353 
00354 static int is_cpp(const char *file_in)
00355 {
00356     size_t l;
00357 
00358     dbg_err_if (file_in == NULL);
00359 
00360     l = strlen(file_in);
00361     if(l < 4)
00362         return 0;
00363 
00364     /* if the file name ends with "[Cc][Cc]" consider it a c++ file */
00365     if(tolower(file_in[--l]) == 'c' && tolower(file_in[--l]) == 'c')
00366         return 1; /* c++ */
00367 
00368 err:
00369     return 0;
00370 }
00371 
00372 static int cb_file(struct dirent *de, const char *path , void *arg)
00373 {
00374     static const char *prefix = "$(srcdir)";
00375     const mime_map_t *mm;
00376     char uri_md5[MD5_DIGEST_BUFSZ];
00377     char file_in[U_FILENAME_MAX], uri[URI_BUFSZ], *base_uri = (char*)arg;
00378     const char *ext;
00379     int enc = 0, zip = 0;
00380 
00381     dbg_err_if (de == NULL);
00382     dbg_err_if (path == NULL);
00383     dbg_err_if (arg == NULL);
00384 
00385     /* input file */
00386     if(path[0] == '/' || path[0] == '\\')
00387     {   /* absolute path */
00388         dbg_err_if(u_snprintf(file_in, U_FILENAME_MAX, "%s/%s", path, 
00389             de->d_name));
00390     } else if(isalpha(path[0]) && path[1] == ':') {
00391         /* absolute path Windows (X:/....) */
00392         dbg_err_if(u_snprintf(file_in, U_FILENAME_MAX, "%s/%s", path, 
00393             de->d_name));
00394     } else {
00395         /* relative path, use $(srcdir) */
00396         dbg_err_if(u_snprintf(file_in, U_FILENAME_MAX, "%s/%s/%s", prefix, 
00397             path, de->d_name));
00398     }
00399 
00400     /* base uri */
00401     dbg_err_if(u_snprintf(uri, URI_BUFSZ, "%s/%s", base_uri, de->d_name));
00402     dbg_err_if(u_md5(uri, strlen(uri), uri_md5));
00403 
00404     /* if the URI match the given exclude pattern then skip it */
00405     if(!pm_is_empty(ctx->excl_patt) && pm_match(ctx->excl_patt, uri))
00406     {
00407         if(ctx->verbose)
00408             con("%s skipped", uri);
00409 
00410         ctx->nexcl++;
00411 
00412         return 0; /* skip it */
00413     } 
00414 
00415     ctx->nfile++;
00416 
00417     /* if the URI match the given encrypt pattern then encrypt it */
00418     if(!pm_is_empty(ctx->enc_patt) && pm_match(ctx->enc_patt, uri))
00419         enc = 1;
00420 
00421     if(ctx->compress) /* -z or -Z have been used */
00422     {
00423         /* if the URI match the given compress pattern then compress it */
00424         if(!pm_is_empty(ctx->comp_patt))
00425         {   /* compression enabled basing on file URI pattern */
00426             if(pm_match(ctx->comp_patt, uri))
00427                 zip = 1;
00428         } else {
00429             /* compression enabled basing on URI MIME types */
00430             if((mm = u_get_mime_map(uri)) != NULL)
00431                 zip = mm->comp;
00432         }
00433     }
00434 
00435     /* print out some info */
00436     if(ctx->verbose == 1)
00437         con("%s (encrypted: %s, compressed: %s)", 
00438             uri, enc ? "yes" : "no", zip ? "yes" : "no");
00439     else if(ctx->verbose > 1)
00440         con("%s -> %s (encrypted: %s, compressed: %s)", 
00441             file_in + strlen(prefix), uri, 
00442             enc ? "yes" : "no", zip ? "yes" : "no");
00443 
00444     ext = u_match_ext(file_in, "klx") ? "cc" : "c";
00445 
00446     dbg_err_if(io_printf(ctx->iom, " \\\n" KL1_FILE_FMT, uri_md5, ext) < 0);
00447 
00448     dbg_err_if(io_printf(ctx->ior, "KLONE_REGISTER(action,%s);\n", uri_md5) <0);
00449 
00450     /* we're adding a '/' before the uri (that will be removed by klone.exe) 
00451      * to avoid MSYS (win32) automatic path translation oddity */
00452     dbg_err_if(io_printf(ctx->iod, 
00453             "\n" KL1_FILE_FMT 
00454             ": %s\n\t$(KLONE) -c translate -i $< -o $@ -u /%s %s %s %s %s\n", 
00455             uri_md5, ext, file_in, uri, 
00456             zip ? "-z" : "",
00457             enc ? "-E" : "", 
00458             enc && ctx->key_file ? "-k" : "", 
00459             enc && ctx->key_file ? ctx->key_file  : ""
00460             ) < 0);
00461 
00462     return 0;
00463 err:
00464     return ~0;
00465 }
00466 
00467 static int cb_dir(struct dirent *de, const char *path , void *arg)
00468 {
00469     char dir[U_FILENAME_MAX], base_uri[URI_BUFSZ], *cur_uri = (char*)arg;
00470 
00471     dbg_err_if (de == NULL);
00472     dbg_err_if (path == NULL);
00473     dbg_err_if (arg == NULL);
00474     
00475     ctx->ndir++;
00476 
00477     dbg_err_if(u_snprintf(dir, U_FILENAME_MAX, "%s/%s", path, de->d_name));
00478 
00479     dbg_err_if(u_snprintf(base_uri, URI_BUFSZ, "%s/%s", cur_uri, de->d_name));
00480 
00481     u_foreach_dir_item(dir, S_IFREG, cb_file, (void*)base_uri);
00482 
00483     u_foreach_dir_item(dir, S_IFDIR, cb_dir, (void*)base_uri);
00484 
00485     return 0;
00486 err:
00487     return ~0;
00488 }
00489 
00490 static int print_register_header(io_t *out)
00491 {
00492     dbg_err_if (out == NULL);
00493  
00494     dbg_err_if(io_printf(out, "#include <klone_conf.h>\n") < 0);
00495     dbg_err_if(io_printf(out, "#include <klone/hook.h>\n") < 0);
00496     dbg_err_if(io_printf(out, "static void do_register(int);\n") < 0);
00497     dbg_err_if(io_printf(out, "void unregister_pages(void);\n") < 0);
00498     dbg_err_if(io_printf(out, "void register_pages(void);\n") < 0);
00499 
00500     dbg_err_if(io_printf(out, 
00501         "void unregister_pages(void) { \n"
00502         "do_register(0); }\n"
00503         ) < 0);
00504     dbg_err_if(io_printf(out, 
00505         "void register_pages(void) { \n"
00506         "do_register(1);\n"
00507         "#ifdef ENABLE_HOOKS\n"
00508         "    hooks_setup(); \n"
00509         "#endif \n"
00510         "}\n") < 0);
00511     dbg_err_if(io_printf(out, 
00512         "static void do_register(int action) {\n") < 0);
00513     dbg_err_if(io_printf(out,
00514         "#define KLONE_REGISTER(a, md5)     \\\n"
00515         "    do {                           \\\n"
00516         "    void module_init_##md5(void);  \\\n"
00517         "    void module_term_##md5(void);  \\\n"
00518         "    if(a) module_init_##md5();     \\\n"
00519         "    else module_term_##md5();      \\\n"
00520         "    } while(0)                     \n") < 0);
00521 
00522     return 0;
00523 err:
00524     return ~0;
00525 }
00526 
00527 static int print_register_footer(io_t *out)
00528 {
00529     dbg_err_if (out == NULL);
00530 
00531     dbg_err_if(io_printf(out, "#undef KLONE_REGISTER\n") < 0);
00532     dbg_err_if(io_printf(out, "}\n") < 0);
00533 
00534     return 0;
00535 err:
00536     return ~0;
00537 }
00538 
00539 static int trans_site(char *root_dir, char *base_uri)
00540 {
00541     dbg_err_if (root_dir == NULL);
00542     dbg_err_if (base_uri == NULL);
00543     
00544     /* makefile */
00545     dbg_err_if(u_file_open("autogen.mk", O_CREAT | O_TRUNC | O_WRONLY, 
00546                 &ctx->iom));
00547     dbg_err_if(u_file_open("autogen.dps", O_CREAT | O_TRUNC | O_WRONLY, 
00548                 &ctx->iod));
00549 
00550     dbg_err_if(u_file_open("register.c", O_CREAT | O_TRUNC | O_WRONLY, 
00551                 &ctx->ior));
00552 
00553     dbg_err_if(print_register_header(ctx->ior));
00554 
00555     dbg_err_if(io_printf(ctx->iom, "embfs_rootdir=%s\n", root_dir) < 0);
00556     dbg_err_if(io_printf(ctx->iom, "autogen_src= ") < 0);
00557 
00558     /* for each file call cb_file */
00559     u_foreach_dir_item(root_dir, S_IFREG, cb_file, base_uri);
00560     /* for each directory call cb_dir */
00561     u_foreach_dir_item(root_dir, S_IFDIR, cb_dir, base_uri);
00562 
00563     dbg_err_if(print_register_footer(ctx->ior));
00564 
00565     io_free(ctx->ior);
00566     io_free(ctx->iod);
00567     io_free(ctx->iom);
00568 
00569     return 0;
00570 err:
00571     if(ctx->ior)
00572         io_free(ctx->ior);
00573     if(ctx->iod)
00574         io_free(ctx->iod);
00575     if(ctx->iom)
00576         io_free(ctx->iom);
00577     return ~0;
00578 }
00579 
00580 static int command_import(void)
00581 {
00582     char *root_dir, *base_uri;
00583 
00584     if(ctx->narg != 1)
00585         usage();    /* just on directory expected */
00586 
00587     root_dir = ctx->arg[0];
00588     dbg_err_if(root_dir == NULL);
00589 
00590     if((base_uri = ctx->base_uri) == NULL)
00591     {
00592         base_uri = u_strdup("");
00593         dbg_err_if (base_uri == NULL);
00594     }
00595 
00596     dbg_err_if(trans_site(root_dir, base_uri));
00597 
00598     con("%lu dirs and %lu files imported, %lu files skipped", 
00599             ctx->ndir, ctx->nfile, ctx->nexcl);
00600 
00601     return 0;
00602 err:
00603     con("import error");
00604     return ~0;
00605 }
00606 
00607 static int dispatch_command(void)
00608 {
00609     switch(ctx->cmd)
00610     {
00611     case CMD_TRANS:
00612         dbg_err_if(command_trans());
00613         break;
00614     case CMD_IMPORT:
00615         dbg_err_if(command_import());
00616         break;
00617     default:
00618         con_err("unknown command");
00619     }
00620 
00621     return 0;
00622 err:
00623     return ~0;
00624 }
00625 
00626 int main(int argc, char **argv)
00627 {
00628     context_t context;
00629 
00630     ctx = &context;
00631 
00632     /* zero-out the context */
00633     memset(ctx, 0, sizeof(context_t));
00634 
00635     /* init pattern matching objects */
00636     dbg_err_if(pm_create(&ctx->comp_patt));
00637     dbg_err_if(pm_create(&ctx->enc_patt));
00638     dbg_err_if(pm_create(&ctx->excl_patt));
00639 
00640     /* parse command line switches and set ctx->cmd and params */
00641     dbg_err_if(parse_opt(argc, argv));
00642 
00643     /* run the command */
00644     dbg_err_if(dispatch_command());
00645 
00646     return EXIT_SUCCESS;
00647 err:
00648     return EXIT_FAILURE;
00649 }