make a character array (hexval) about 10 characters long
set whole array to spaces
set hexval[9] to '\0'
set hexptr to 8 -- this will point into hexval to load the hex chars.
Read a line into an integer (intval) -- that's base 10.
Loop until that integer is 0:
    set hexval[hexptr] = remainder of intval/16 
    add 48 ('0') to hexval[hexptr]  -- this converts the integer value to a digit character
    if hexval[hexptr] > 57 ('9') add 7 -- value is now 'A' - 'F'
    decrement hexptr  -- to load the next hex digit
    divide 16 from intval -- to remove the hex val we just loaded
    End of loop

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
char dek_hex(int);         /*a function I'll explain later*/
main(){
FILE *a,*b;
int broj;
a=fopen("C:\\C_fajlovi\\deci.txt","r");         /*created a file in the specified directory, it's all cool at this point*/
b=fopen("C:\\C_fajlovi\\hex.txt","w");             /*create a new file to write to*/
while(fscanf(a,"%d",&broj)!=EOF)
 fprintf(b,"%s",&dek_hex(broj));        /*dek_hex should be a function that turns a dec number into hexadecimal and returns it in the form of a string*/
fcloseall();
}
char dek_hex(int m){             /*this is where my problems start. No matter how I write the function, I keep getting all kinds of error messages*/

FILE* fp ;
if( ( fp = fopen( "a.txt", "r" ) ) == NULL )
{
     fputs( "file opening failed", stderr ) ;
     exit( 1 ) ;
}

char buf[20];
int i = 16;
sprintf(buf,"%x",i);