[svn-commits] twilson: branch twilson/minimime-memory-usage r104034 - in /team/twilson/mini...
SVN commits to the Digium repositories
svn-commits at lists.digium.com
Fri Feb 22 11:09:15 CST 2008
Author: twilson
Date: Fri Feb 22 11:09:14 2008
New Revision: 104034
URL: http://svn.digium.com/view/asterisk?view=rev&rev=104034
Log:
Initial imput of changes to make minimime not malloc space for an entire upload.
This uses tmpfile() and stores a pointer to the file in struct mm_mimepart. But in
testing, I've noticed that minimime chokes on files somehwere between 20 and 39 MB +
and errors out saying it can't allocate more space in the imput buffer because the
scanner is using REJECT. Assuming this is stack space(I know next to nothing about
flex), but have asked murf to take a look at it and see if there is something easy
relatively easy to do about it.
Modified:
team/twilson/minimime-memory-usage/ (props changed)
team/twilson/minimime-memory-usage/main/http.c
team/twilson/minimime-memory-usage/main/minimime/mimeparser.tab.c
team/twilson/minimime-memory-usage/main/minimime/mimeparser.y
team/twilson/minimime-memory-usage/main/minimime/mimeparser.yy.c
team/twilson/minimime-memory-usage/main/minimime/mm.h
team/twilson/minimime-memory-usage/main/minimime/mm_mimepart.c
Propchange: team/twilson/minimime-memory-usage/
------------------------------------------------------------------------------
automerge = *
Propchange: team/twilson/minimime-memory-usage/
------------------------------------------------------------------------------
svnmerge-integrated = /trunk:1-104032
Modified: team/twilson/minimime-memory-usage/main/http.c
URL: http://svn.digium.com/view/asterisk/team/twilson/minimime-memory-usage/main/http.c?view=diff&rev=104034&r1=104033&r2=104034
==============================================================================
--- team/twilson/minimime-memory-usage/main/http.c (original)
+++ team/twilson/minimime-memory-usage/main/http.c Fri Feb 22 11:09:14 2008
@@ -365,9 +365,10 @@
static void post_raw(struct mm_mimepart *part, const char *post_dir, const char *fn)
{
char filename[PATH_MAX];
- FILE *f;
- const char *body;
- size_t body_len;
+ FILE *f, *body;
+ size_t len;
+ int res;
+ char buf[4096];
snprintf(filename, sizeof(filename), "%s/%s", post_dir, fn);
@@ -378,25 +379,30 @@
return;
}
- if (!(body = mm_mimepart_getbody(part, 0))) {
- ast_debug(1, "Couldn't get the mimepart body\n");
+ if (!(body = mm_mimepart_getbody_file(part, 0))) {
+ ast_log(LOG_WARNING, "Couldn't get the mimepart body\n");
fclose(f);
return;
}
- body_len = mm_mimepart_getlength(part);
-
- ast_debug(1, "Body length is %ld\n", (long int)body_len);
-
- fwrite(body, 1, body_len, f);
+ len = mm_mimepart_getlength(part);
+ ast_debug(1, "Body is %lu and %lu long\n", (unsigned long)body, len);
+
+ for(res = sizeof(buf);len;len -= res) {
+ if (len < res)
+ res = len;
+ fread(buf, 1, res, body);
+ fwrite(buf, 1, res, f);
+ }
fclose(f);
+ fclose(body);
}
static struct ast_str *handle_post(struct server_instance *ser, char *uri,
int *status, char **title, int *contentlength, struct ast_variable *headers,
struct ast_variable *cookies)
{
- char buf;
+ char buf[4096];
FILE *f;
size_t res;
struct ast_variable *var;
@@ -446,11 +452,11 @@
fprintf(f, "Content-Type: %s\r\n\r\n", var->value);
}
- while ((res = fread(&buf, 1, 1, ser->f))) {
- fwrite(&buf, 1, 1, f);
- content_len--;
- if (!content_len)
- break;
+ for(res = sizeof(buf);content_len;content_len -= res) {
+ if (content_len < res)
+ res = content_len;
+ fread(buf, 1, res, ser->f);
+ fwrite(buf, 1, res, f);
}
if (fseek(f, SEEK_SET, 0)) {
Modified: team/twilson/minimime-memory-usage/main/minimime/mimeparser.tab.c
URL: http://svn.digium.com/view/asterisk/team/twilson/minimime-memory-usage/main/minimime/mimeparser.tab.c?view=diff&rev=104034&r1=104033&r2=104034
==============================================================================
--- team/twilson/minimime-memory-usage/main/minimime/mimeparser.tab.c (original)
+++ team/twilson/minimime-memory-usage/main/minimime/mimeparser.tab.c Fri Feb 22 11:09:14 2008
@@ -184,7 +184,7 @@
void reset_environ(struct parser_state *pstate);
int PARSER_initialize(struct parser_state *pstate, void *yyscanner);
-static char *PARSE_readmessagepart(size_t, size_t, size_t, size_t *,yyscan_t, struct parser_state *);
+static char *PARSE_readmessagepart(size_t, size_t, size_t, size_t *,yyscan_t, struct parser_state *, FILE *tempfile);
FILE *mimeparser_yyget_in (yyscan_t yyscanner );
@@ -279,7 +279,7 @@
#define YYSIZE_MAXIMUM ((YYSIZE_T) -1)
#ifndef YY_
-# if defined(YYENABLE_NLS)
+# if YYENABLE_NLS
# if ENABLE_NLS
# include <libintl.h> /* INFRINGES ON USER NAME SPACE */
# define YY_(msgid) dgettext ("bison-runtime", msgid)
@@ -757,7 +757,7 @@
we won't break user code: when these are the locations we know. */
#ifndef YY_LOCATION_PRINT
-# if defined(YYLTYPE_IS_TRIVIAL)
+# if YYLTYPE_IS_TRIVIAL
# define YY_LOCATION_PRINT(File, Loc) \
fprintf (File, "%d.%d-%d.%d", \
(Loc).first_line, (Loc).first_column, \
@@ -1568,7 +1568,7 @@
if ((yyvsp[(1) - (1)].position).start != (yyvsp[(1) - (1)].position).end) {
preamble = PARSE_readmessagepart(0, (yyvsp[(1) - (1)].position).start, (yyvsp[(1) - (1)].position).end,
- &offset,yyscanner,pstate);
+ &offset,yyscanner,pstate,NULL);
if (preamble == NULL) {
return(-1);
}
@@ -1891,25 +1891,36 @@
{
char *body;
size_t offset;
+ FILE *body_fileptr = NULL;
+ const char *filename = NULL;
+
+ filename = mm_content_getdispositionparambyname(pstate->current_mimepart->type, "filename");
+
+ if (filename && strlen(filename) && !(body_fileptr = tmpfile()))
+ return(-1);
dprintf2(pstate,"BODY (%d/%d), SIZE %d\n", (yyvsp[(1) - (1)].position).start, (yyvsp[(1) - (1)].position).end, (yyvsp[(1) - (1)].position).end - (yyvsp[(1) - (1)].position).start);
body = PARSE_readmessagepart((yyvsp[(1) - (1)].position).opaque_start, (yyvsp[(1) - (1)].position).start, (yyvsp[(1) - (1)].position).end,
- &offset,yyscanner,pstate);
-
- if (body == NULL) {
+ &offset,yyscanner,pstate,body_fileptr);
+
+ if ((!filename || !strlen(filename)) && body == NULL) {
return(-1);
}
pstate->current_mimepart->opaque_body = body;
- pstate->current_mimepart->body = body + offset;
+
+ if (body)
+ pstate->current_mimepart->body = body + offset;
+
pstate->current_mimepart->opaque_length = (yyvsp[(1) - (1)].position).end - (yyvsp[(1) - (1)].position).start - 2 + offset;
pstate->current_mimepart->length = pstate->current_mimepart->opaque_length - offset;
+ pstate->current_mimepart->body_fileptr = body_fileptr;
;}
break;
/* Line 1267 of yacc.c. */
-#line 1913 "mimeparser.tab.c"
+#line 1924 "mimeparser.tab.c"
default: break;
}
YY_SYMBOL_PRINT ("-> $$ =", yyr1[yyn], &yyval, &yyloc);
@@ -2123,7 +2134,7 @@
}
-#line 537 "mimeparser.y"
+#line 548 "mimeparser.y"
/*
@@ -2131,7 +2142,7 @@
*/
static char *
PARSE_readmessagepart(size_t opaque_start, size_t real_start, size_t end,
- size_t *offset, yyscan_t yyscanner, struct parser_state *pstate)
+ size_t *offset, yyscan_t yyscanner, struct parser_state *pstate, FILE *tempfile)
{
size_t body_size;
size_t current;
@@ -2196,11 +2207,15 @@
return(NULL);
}
- body = (char *)malloc(body_size + 1);
- if (body == NULL) {
- mm_errno = MM_ERROR_ERRNO;
- return(NULL);
- }
+ if (!tempfile) {
+ body = (char *)malloc(body_size + 1);
+ if (body == NULL) {
+ mm_errno = MM_ERROR_ERRNO;
+ return(NULL);
+ }
+ } else {
+ body = NULL;
+ }
/* Get the message body either from a stream or a memory
* buffer.
@@ -2209,7 +2224,22 @@
FILE *x = mimeparser_yyget_in(yyscanner);
current = ftell(x);
fseek(x, start - 1, SEEK_SET);
- fread(body, body_size - 1, 1, x);
+ if (!tempfile) {
+ fread(body, body_size - 1, 1, x);
+ } else {
+ char buf[4096];
+ int res, len = end - start - 2 - *offset;
+
+ fseek(x, *offset, SEEK_CUR);
+
+ for(res = sizeof(buf);len;len -= res) {
+ if (len < res)
+ res = len;
+ fread(buf, 1, res, x);
+ fwrite(buf, 1, res, tempfile);
+ }
+ fseek(tempfile, 0L, SEEK_SET);
+ }
fseek(x, current, SEEK_SET);
} else if (pstate->lstate.message_buffer != NULL) {
strlcpy(body, pstate->lstate.message_buffer + start - 1, body_size);
Modified: team/twilson/minimime-memory-usage/main/minimime/mimeparser.y
URL: http://svn.digium.com/view/asterisk/team/twilson/minimime-memory-usage/main/minimime/mimeparser.y?view=diff&rev=104034&r1=104033&r2=104034
==============================================================================
--- team/twilson/minimime-memory-usage/main/minimime/mimeparser.y (original)
+++ team/twilson/minimime-memory-usage/main/minimime/mimeparser.y Fri Feb 22 11:09:14 2008
@@ -53,7 +53,7 @@
void reset_environ(struct parser_state *pstate);
int PARSER_initialize(struct parser_state *pstate, void *yyscanner);
-static char *PARSE_readmessagepart(size_t, size_t, size_t, size_t *,yyscan_t, struct parser_state *);
+static char *PARSE_readmessagepart(size_t, size_t, size_t, size_t *,yyscan_t, struct parser_state *, FILE *tempfile);
FILE *mimeparser_yyget_in (yyscan_t yyscanner );
%}
@@ -173,7 +173,7 @@
if ($1.start != $1.end) {
preamble = PARSE_readmessagepart(0, $1.start, $1.end,
- &offset,yyscanner,pstate);
+ &offset,yyscanner,pstate,NULL);
if (preamble == NULL) {
return(-1);
}
@@ -518,19 +518,30 @@
{
char *body;
size_t offset;
+ FILE *body_fileptr = NULL;
+ const char *filename = NULL;
+
+ filename = mm_content_getdispositionparambyname(pstate->current_mimepart->type, "filename");
+
+ if (filename && strlen(filename) && !(body_fileptr = tmpfile()))
+ return(-1);
dprintf2(pstate,"BODY (%d/%d), SIZE %d\n", $1.start, $1.end, $1.end - $1.start);
body = PARSE_readmessagepart($1.opaque_start, $1.start, $1.end,
- &offset,yyscanner,pstate);
-
- if (body == NULL) {
+ &offset,yyscanner,pstate,body_fileptr);
+
+ if ((!filename || !strlen(filename)) && body == NULL) {
return(-1);
}
pstate->current_mimepart->opaque_body = body;
- pstate->current_mimepart->body = body + offset;
+
+ if (body)
+ pstate->current_mimepart->body = body + offset;
+
pstate->current_mimepart->opaque_length = $1.end - $1.start - 2 + offset;
pstate->current_mimepart->length = pstate->current_mimepart->opaque_length - offset;
+ pstate->current_mimepart->body_fileptr = body_fileptr;
}
;
@@ -541,7 +552,7 @@
*/
static char *
PARSE_readmessagepart(size_t opaque_start, size_t real_start, size_t end,
- size_t *offset, yyscan_t yyscanner, struct parser_state *pstate)
+ size_t *offset, yyscan_t yyscanner, struct parser_state *pstate, FILE *tempfile)
{
size_t body_size;
size_t current;
@@ -606,11 +617,15 @@
return(NULL);
}
- body = (char *)malloc(body_size + 1);
- if (body == NULL) {
- mm_errno = MM_ERROR_ERRNO;
- return(NULL);
- }
+ if (!tempfile) {
+ body = (char *)malloc(body_size + 1);
+ if (body == NULL) {
+ mm_errno = MM_ERROR_ERRNO;
+ return(NULL);
+ }
+ } else {
+ body = NULL;
+ }
/* Get the message body either from a stream or a memory
* buffer.
@@ -619,7 +634,22 @@
FILE *x = mimeparser_yyget_in(yyscanner);
current = ftell(x);
fseek(x, start - 1, SEEK_SET);
- fread(body, body_size - 1, 1, x);
+ if (!tempfile) {
+ fread(body, body_size - 1, 1, x);
+ } else {
+ char buf[4096];
+ int res, len = end - start - 2 - *offset;
+
+ fseek(x, *offset, SEEK_CUR);
+
+ for(res = sizeof(buf);len;len -= res) {
+ if (len < res)
+ res = len;
+ fread(buf, 1, res, x);
+ fwrite(buf, 1, res, tempfile);
+ }
+ fseek(tempfile, 0L, SEEK_SET);
+ }
fseek(x, current, SEEK_SET);
} else if (pstate->lstate.message_buffer != NULL) {
strlcpy(body, pstate->lstate.message_buffer + start - 1, body_size);
Modified: team/twilson/minimime-memory-usage/main/minimime/mimeparser.yy.c
URL: http://svn.digium.com/view/asterisk/team/twilson/minimime-memory-usage/main/minimime/mimeparser.yy.c?view=diff&rev=104034&r1=104033&r2=104034
==============================================================================
--- team/twilson/minimime-memory-usage/main/minimime/mimeparser.yy.c (original)
+++ team/twilson/minimime-memory-usage/main/minimime/mimeparser.yy.c Fri Feb 22 11:09:14 2008
@@ -2549,7 +2549,8 @@
void mimeparser_yyfree (void * ptr , yyscan_t yyscanner)
{
- free( (char *) ptr ); /* see mimeparser_yyrealloc() for (char *) cast */
+ if (ptr)
+ free( (char *) ptr ); /* see mimeparser_yyrealloc() for (char *) cast */
}
#define YYTABLES_NAME "yytables"
Modified: team/twilson/minimime-memory-usage/main/minimime/mm.h
URL: http://svn.digium.com/view/asterisk/team/twilson/minimime-memory-usage/main/minimime/mm.h?view=diff&rev=104034&r1=104033&r2=104034
==============================================================================
--- team/twilson/minimime-memory-usage/main/minimime/mm.h (original)
+++ team/twilson/minimime-memory-usage/main/minimime/mm.h Fri Feb 22 11:09:14 2008
@@ -224,6 +224,7 @@
size_t length;
char *body;
+ FILE *body_fileptr;
struct mm_content *type;
@@ -298,6 +299,7 @@
struct mm_content *mm_mimepart_getcontent(struct mm_mimepart *);
size_t mm_mimepart_getlength(struct mm_mimepart *);
char *mm_mimepart_getbody(struct mm_mimepart *, int);
+FILE *mm_mimepart_getbody_file(struct mm_mimepart *, int);
void mm_mimepart_attachcontenttype(struct mm_mimepart *, struct mm_content *);
int mm_mimepart_setdefaultcontenttype(struct mm_mimepart *, int);
int mm_mimepart_flatten(struct mm_mimepart *, char **, size_t *, int);
Modified: team/twilson/minimime-memory-usage/main/minimime/mm_mimepart.c
URL: http://svn.digium.com/view/asterisk/team/twilson/minimime-memory-usage/main/minimime/mm_mimepart.c?view=diff&rev=104034&r1=104033&r2=104034
==============================================================================
--- team/twilson/minimime-memory-usage/main/minimime/mm_mimepart.c (original)
+++ team/twilson/minimime-memory-usage/main/minimime/mm_mimepart.c Fri Feb 22 11:09:14 2008
@@ -395,6 +395,14 @@
return part->opaque_body;
else
return part->body;
+}
+
+FILE *
+mm_mimepart_getbody_file(struct mm_mimepart *part, int opaque)
+{
+ assert(part != NULL);
+
+ return part->body_fileptr;
}
/**
More information about the svn-commits
mailing list