fopen.c

Go to the documentation of this file.

/*
 * fopen.c
 *
 * adapted from https://curl.haxx.se/libcurl/c/fopen.html.
 *
 * Modifications to read/write wide character streams by
 * Simon Brooke.
 *
 * NOTE THAT: for my purposes, I'm only interested in wide characters,
 * and I always read them one character at a time.
 *
 * Copyright (c) 2003, 2017 Simtec Electronics
 * Some portions (c) 2019 Simon Brooke <simon@journeyman.cc>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. The name of the author may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * This example requires libcurl 7.9.7 or later.
 */
 
#include <stdio.h>
#include <string.h>
#ifndef WIN32
#include <sys/time.h>
#endif
#include <stdlib.h>
#include <errno.h>
 
#include <curl/curl.h>
 
#include "io/fopen.h"
#ifdef FOPEN_STANDALONE
CURLSH *io_share;
#else
#include "memory/consspaceobject.h"
#include "io/io.h"
#include "utils.h"
#endif
 
 
/* exported functions */
URL_FILE *url_fopen( const char *url, const char *operation );
int url_fclose( URL_FILE * file );
int url_feof( URL_FILE * file );
size_t url_fread( void *ptr, size_t size, size_t nmemb, URL_FILE * file );
char *url_fgets( char *ptr, size_t size, URL_FILE * file );
void url_rewind( URL_FILE * file );
 
/* we use a global one for convenience */
static CURLM *multi_handle;
 
/* curl calls this routine to get more data */
static size_t write_callback( char *buffer,
                              size_t size, size_t nitems, void *userp ) {
    char *newbuff;
    size_t rembuff;
 
    URL_FILE *url = ( URL_FILE * ) userp;
    size *= nitems;
 
    rembuff = url->buffer_len - url->buffer_pos;  /* remaining space in buffer */
 
    if ( size > rembuff ) {
        /* not enough space in buffer */
        newbuff = realloc( url->buffer, url->buffer_len + ( size - rembuff ) );
        if ( newbuff == NULL ) {
            fprintf( stderr, "callback buffer grow failed\n" );
            size = rembuff;
        } else {
            /* realloc succeeded increase buffer size */
            url->buffer_len += size - rembuff;
            url->buffer = newbuff;
        }
    }
 
    memcpy( &url->buffer[url->buffer_pos], buffer, size );
    url->buffer_pos += size;
 
    return size;
}
 
/* use to attempt to fill the read buffer up to requested number of bytes */
static int fill_buffer( URL_FILE *file, size_t want ) {
    fd_set fdread;
    fd_set fdwrite;
    fd_set fdexcep;
    struct timeval timeout;
    int rc;
    CURLMcode mc;               /* curl_multi_fdset() return code */
 
    /* only attempt to fill buffer if transactions still running and buffer
     * doesn't exceed required size already
     */
    if ( ( !file->still_running ) || ( file->buffer_pos > want ) )
        return 0;
 
    /* attempt to fill buffer */
    do {
        int maxfd = -1;
        long curl_timeo = -1;
 
        FD_ZERO( &fdread );
        FD_ZERO( &fdwrite );
        FD_ZERO( &fdexcep );
 
        /* set a suitable timeout to fail on */
        timeout.tv_sec = 60;    /* 1 minute */
        timeout.tv_usec = 0;
 
        curl_multi_timeout( multi_handle, &curl_timeo );
        if ( curl_timeo >= 0 ) {
            timeout.tv_sec = curl_timeo / 1000;
            if ( timeout.tv_sec > 1 )
                timeout.tv_sec = 1;
            else
                timeout.tv_usec = ( curl_timeo % 1000 ) * 1000;
        }
 
        /* get file descriptors from the transfers */
        mc = curl_multi_fdset( multi_handle, &fdread, &fdwrite, &fdexcep,
                               &maxfd );
 
        if ( mc != CURLM_OK ) {
            fprintf( stderr, "curl_multi_fdset() failed, code %d.\n", mc );
            break;
        }
 
        /* On success the value of maxfd is guaranteed to be >= -1. We call
           select(maxfd + 1, ...); specially in case of (maxfd == -1) there are
           no fds ready yet so we call select(0, ...) --or Sleep() on Windows--
           to sleep 100ms, which is the minimum suggested value in the
           curl_multi_fdset() doc. */
 
        if ( maxfd == -1 ) {
#ifdef _WIN32
            Sleep( 100 );
            rc = 0;
#else
            /* Portable sleep for platforms other than Windows. */
            struct timeval wait = { 0, 100 * 1000 };  /* 100ms */
            rc = select( 0, NULL, NULL, NULL, &wait );
#endif
        } else {
            /* Note that on some platforms 'timeout' may be modified by select().
               If you need access to the original value save a copy beforehand. */
            rc = select( maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout );
        }
 
        switch ( rc ) {
            case -1:
                /* select error */
                break;
 
            case 0:
            default:
                /* timeout or readable/writable sockets */
                curl_multi_perform( multi_handle, &file->still_running );
                break;
        }
    } while ( file->still_running && ( file->buffer_pos < want ) );
    return 1;
}
 
/* use to remove want bytes from the front of a files buffer */
static int use_buffer( URL_FILE *file, size_t want ) {
    /* sort out buffer */
    if ( ( file->buffer_pos - want ) <= 0 ) {
        /* ditch buffer - write will recreate */
        free( file->buffer );
        file->buffer = NULL;
        file->buffer_pos = 0;
        file->buffer_len = 0;
    } else {
        /* move rest down make it available for later */
        memmove( file->buffer,
                 &file->buffer[want], ( file->buffer_pos - want ) );
 
        file->buffer_pos -= want;
    }
    return 0;
}
 
URL_FILE *url_fopen( const char *url, const char *operation ) {
    /* this code could check for URLs or types in the 'url' and
       basically use the real fopen() for standard files */
 
    URL_FILE *file;
    ( void ) operation;
 
    file = calloc( 1, sizeof( URL_FILE ) );
    if ( !file )
        return NULL;
 
    file->handle.file = fopen( url, operation );
    if ( file->handle.file ) {
        file->type = CFTYPE_FILE; /* marked as file */
    } else if ( index_of( ':', url ) > -1 ) {
        file->type = CFTYPE_CURL; /* marked as URL */
        file->handle.curl = curl_easy_init(  );
 
        curl_easy_setopt( file->handle.curl, CURLOPT_URL, url );
        curl_easy_setopt( file->handle.curl, CURLOPT_WRITEDATA, file );
        curl_easy_setopt( file->handle.curl, CURLOPT_VERBOSE, 0L );
        curl_easy_setopt( file->handle.curl, CURLOPT_WRITEFUNCTION,
                          write_callback );
        /* use the share object */
        curl_easy_setopt( file->handle.curl, CURLOPT_SHARE, io_share );
 
 
        if ( !multi_handle )
            multi_handle = curl_multi_init(  );
 
        curl_multi_add_handle( multi_handle, file->handle.curl );
 
        /* lets start the fetch */
        curl_multi_perform( multi_handle, &file->still_running );
 
        if ( ( file->buffer_pos == 0 ) && ( !file->still_running ) ) {
            /* if still_running is 0 now, we should return NULL */
 
            /* make sure the easy handle is not in the multi handle anymore */
            curl_multi_remove_handle( multi_handle, file->handle.curl );
 
            /* cleanup */
            curl_easy_cleanup( file->handle.curl );
 
            free( file );
 
            file = NULL;
        }
    } else {
        file->type = CFTYPE_NONE;
        /* not a file, and doesn't look like a URL. */
    }
 
    return file;
}
 
int url_fclose( URL_FILE *file ) {
    int ret = 0;                /* default is good return */
 
    switch ( file->type ) {
        case CFTYPE_FILE:
            ret = fclose( file->handle.file );  /* passthrough */
            break;
 
        case CFTYPE_CURL:
            /* make sure the easy handle is not in the multi handle anymore */
            curl_multi_remove_handle( multi_handle, file->handle.curl );
 
            /* cleanup */
            curl_easy_cleanup( file->handle.curl );
            break;
 
        default:               /* unknown or supported type - oh dear */
            ret = EOF;
            errno = EBADF;
            break;
    }
 
    free( file->buffer );       /* free any allocated buffer space */
    free( file );
 
    return ret;
}
 
int url_feof( URL_FILE *file ) {
    int ret = 0;
 
    switch ( file->type ) {
        case CFTYPE_FILE:
            ret = feof( file->handle.file );
            break;
 
        case CFTYPE_CURL:
            if ( ( file->buffer_pos == 0 ) && ( !file->still_running ) )
                ret = 1;
            break;
 
        default:               /* unknown or supported type - oh dear */
            ret = -1;
            errno = EBADF;
            break;
    }
    return ret;
}
 
size_t url_fread( void *ptr, size_t size, size_t nmemb, URL_FILE *file ) {
    size_t want;
 
    switch ( file->type ) {
        case CFTYPE_FILE:
            want = fread( ptr, size, nmemb, file->handle.file );
            break;
 
        case CFTYPE_CURL:
            want = nmemb * size;
 
            fill_buffer( file, want );
 
            /* check if there's data in the buffer - if not fill_buffer()
             * either errored or EOF */
            if ( !file->buffer_pos )
                return 0;
 
            /* ensure only available data is considered */
            if ( file->buffer_pos < want )
                want = file->buffer_pos;
 
            /* xfer data to caller */
            memcpy( ptr, file->buffer, want );
 
            use_buffer( file, want );
 
            want = want / size; /* number of items */
            break;
 
        default:               /* unknown or supported type - oh dear */
            want = 0;
            errno = EBADF;
            break;
 
    }
    return want;
}
 
char *url_fgets( char *ptr, size_t size, URL_FILE *file ) {
    size_t want = size - 1;     /* always need to leave room for zero termination */
    size_t loop;
 
    switch ( file->type ) {
        case CFTYPE_FILE:
            ptr = fgets( ptr, ( int ) size, file->handle.file );
            break;
 
        case CFTYPE_CURL:
            fill_buffer( file, want );
 
            /* check if there's data in the buffer - if not fill either errored or
             * EOF */
            if ( !file->buffer_pos )
                return NULL;
 
            /* ensure only available data is considered */
            if ( file->buffer_pos < want )
                want = file->buffer_pos;
 
            /*buffer contains data */
            /* look for newline or eof */
            for ( loop = 0; loop < want; loop++ ) {
                if ( file->buffer[loop] == '\n' ) {
                    want = loop + 1;  /* include newline */
                    break;
                }
            }
 
            /* xfer data to caller */
            memcpy( ptr, file->buffer, want );
            ptr[want] = 0;      /* always null terminate */
 
            use_buffer( file, want );
 
            break;
 
        default:               /* unknown or supported type - oh dear */
            ptr = NULL;
            errno = EBADF;
            break;
    }
 
    return ptr;                 /*success */
}
 
void url_rewind( URL_FILE *file ) {
    switch ( file->type ) {
        case CFTYPE_FILE:
            rewind( file->handle.file );  /* passthrough */
            break;
 
        case CFTYPE_CURL:
            /* halt transaction */
            curl_multi_remove_handle( multi_handle, file->handle.curl );
 
            /* restart */
            curl_multi_add_handle( multi_handle, file->handle.curl );
 
            /* ditch buffer - write will recreate - resets stream pos */
            free( file->buffer );
            file->buffer = NULL;
            file->buffer_pos = 0;
            file->buffer_len = 0;
 
            break;
 
        default:               /* unknown or supported type - oh dear */
            break;
    }
}
 
#ifdef FOPEN_STANDALONE
#define FGETSFILE "fgets.test"
#define FREADFILE "fread.test"
#define REWINDFILE "rewind.test"
 
/* Small main program to retrieve from a url using fgets and fread saving the
 * output to two test files (note the fgets method will corrupt binary files if
 * they contain 0 chars */
int main( int argc, char *argv[] ) {
    URL_FILE *handle;
    FILE *outf;
 
    size_t nread;
    char buffer[256];
    const char *url;
 
    CURL *curl;
    CURLcode res;
 
    curl_global_init( CURL_GLOBAL_DEFAULT );
 
    curl = curl_easy_init(  );
 
 
    if ( argc < 2 )
        url = "http://192.168.7.3/testfile";  /* default to testurl */
    else
        url = argv[1];          /* use passed url */
 
    /* copy from url line by line with fgets */
    outf = fopen( FGETSFILE, "wb+" );
    if ( !outf ) {
        perror( "couldn't open fgets output file\n" );
        return 1;
    }
 
    handle = url_fopen( url, "r" );
    if ( !handle ) {
        printf( "couldn't url_fopen() %s\n", url );
        fclose( outf );
        return 2;
    }
 
    while ( !url_feof( handle ) ) {
        url_fgets( buffer, sizeof( buffer ), handle );
        fwrite( buffer, 1, strlen( buffer ), outf );
    }
 
    url_fclose( handle );
 
    fclose( outf );
 
 
    /* Copy from url with fread */
    outf = fopen( FREADFILE, "wb+" );
    if ( !outf ) {
        perror( "couldn't open fread output file\n" );
        return 1;
    }
 
    handle = url_fopen( "testfile", "r" );
    if ( !handle ) {
        printf( "couldn't url_fopen() testfile\n" );
        fclose( outf );
        return 2;
    }
 
    do {
        nread = url_fread( buffer, 1, sizeof( buffer ), handle );
        fwrite( buffer, 1, nread, outf );
    } while ( nread );
 
    url_fclose( handle );
 
    fclose( outf );
 
 
    /* Test rewind */
    outf = fopen( REWINDFILE, "wb+" );
    if ( !outf ) {
        perror( "couldn't open fread output file\n" );
        return 1;
    }
 
    handle = url_fopen( "testfile", "r" );
    if ( !handle ) {
        printf( "couldn't url_fopen() testfile\n" );
        fclose( outf );
        return 2;
    }
 
    nread = url_fread( buffer, 1, sizeof( buffer ), handle );
    fwrite( buffer, 1, nread, outf );
    url_rewind( handle );
 
    buffer[0] = '\n';
    fwrite( buffer, 1, 1, outf );
 
    nread = url_fread( buffer, 1, sizeof( buffer ), handle );
    fwrite( buffer, 1, nread, outf );
 
    url_fclose( handle );
 
    fclose( outf );
 
    return 0;                   /* all done */
}
#endif