To what extent does the system endianness effect a program. I know that doing pointer assignment will be affected, and unions are too, but what about bitshift operators? Do they apply to the value, merely emulating real bitshifts, or do they apply to the bits themselves? Are there any other cases where endianness should be taken into account?

Endianness only matters when you make an assumption about how bytes are organized. Bitwise operators will work with the value and have consistent behavior regardless of the endianness of the system, and that's one way to ensure portable code when sending or receiving binary data:

#include <algorithm>
#include <iostream>
#include <iterator>
#include <limits>
#include <type_traits>
#include <vector>

using namespace std;

template <typename T>
vector<unsigned char> GetBytesNonPortable(T value, typename enable_if<is_integral<T>::value>::type * = nullptr)
    // Extract bytes as stored in memory (different endianness will give different results)
    unsigned char* pun = (unsigned char*)&value;

    return vector<unsigned char>(pun, pun + sizeof(T));


template <typename T>
vector<unsigned char> GetBytesPortable(T value, typename enable_if<is_integral<T>::value>::type * = nullptr)
    unsigned const bits = numeric_limits<unsigned char>::digits;
    unsigned const max = numeric_limits<unsigned char>::max();

    vector<unsigned char> result;

    // Extract bytes regardless of endianness
    for (size_t i = 0; i < sizeof(T); i++)
        result.push_back((value >> (bits * i)) & max);

    return result;

int main()
    int value = 12345678;
    vector<unsigned char> v1 = GetBytesNonPortable(value);
    vector<unsigned char> v2 = GetBytesPortable(value);

    copy(v1.begin(), v1.end(), ostream_iterator<int>(cout, " ")), cout << endl;
    copy(v2.begin(), v2.end(), ostream_iterator<int>(cout, " ")), cout << endl;

Thank you! That was what I suspected, very helpful!